{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we use the cancer dataset to learn logistic regression."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "#from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_breast_cancer()\n",
    "data = pd.DataFrame(data=dataset.data, columns=dataset.feature_names)\n",
    "data['cancer'] = [dataset.target_names[t] for t in dataset.target]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['malignant' 'benign']\n"
     ]
    }
   ],
   "source": [
    "print(dataset.target_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean radius</th>\n",
       "      <th>mean texture</th>\n",
       "      <th>mean perimeter</th>\n",
       "      <th>mean area</th>\n",
       "      <th>mean smoothness</th>\n",
       "      <th>mean compactness</th>\n",
       "      <th>mean concavity</th>\n",
       "      <th>mean concave points</th>\n",
       "      <th>mean symmetry</th>\n",
       "      <th>mean fractal dimension</th>\n",
       "      <th>...</th>\n",
       "      <th>worst texture</th>\n",
       "      <th>worst perimeter</th>\n",
       "      <th>worst area</th>\n",
       "      <th>worst smoothness</th>\n",
       "      <th>worst compactness</th>\n",
       "      <th>worst concavity</th>\n",
       "      <th>worst concave points</th>\n",
       "      <th>worst symmetry</th>\n",
       "      <th>worst fractal dimension</th>\n",
       "      <th>cancer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19.810</td>\n",
       "      <td>22.15</td>\n",
       "      <td>130.00</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>0.09831</td>\n",
       "      <td>0.10270</td>\n",
       "      <td>0.14790</td>\n",
       "      <td>0.09498</td>\n",
       "      <td>0.1582</td>\n",
       "      <td>0.05395</td>\n",
       "      <td>...</td>\n",
       "      <td>30.88</td>\n",
       "      <td>186.80</td>\n",
       "      <td>2398.0</td>\n",
       "      <td>0.1512</td>\n",
       "      <td>0.3150</td>\n",
       "      <td>0.53720</td>\n",
       "      <td>0.23880</td>\n",
       "      <td>0.2768</td>\n",
       "      <td>0.07615</td>\n",
       "      <td>malignant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>13.540</td>\n",
       "      <td>14.36</td>\n",
       "      <td>87.46</td>\n",
       "      <td>566.3</td>\n",
       "      <td>0.09779</td>\n",
       "      <td>0.08129</td>\n",
       "      <td>0.06664</td>\n",
       "      <td>0.04781</td>\n",
       "      <td>0.1885</td>\n",
       "      <td>0.05766</td>\n",
       "      <td>...</td>\n",
       "      <td>19.26</td>\n",
       "      <td>99.70</td>\n",
       "      <td>711.2</td>\n",
       "      <td>0.1440</td>\n",
       "      <td>0.1773</td>\n",
       "      <td>0.23900</td>\n",
       "      <td>0.12880</td>\n",
       "      <td>0.2977</td>\n",
       "      <td>0.07259</td>\n",
       "      <td>benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>13.080</td>\n",
       "      <td>15.71</td>\n",
       "      <td>85.63</td>\n",
       "      <td>520.0</td>\n",
       "      <td>0.10750</td>\n",
       "      <td>0.12700</td>\n",
       "      <td>0.04568</td>\n",
       "      <td>0.03110</td>\n",
       "      <td>0.1967</td>\n",
       "      <td>0.06811</td>\n",
       "      <td>...</td>\n",
       "      <td>20.49</td>\n",
       "      <td>96.09</td>\n",
       "      <td>630.5</td>\n",
       "      <td>0.1312</td>\n",
       "      <td>0.2776</td>\n",
       "      <td>0.18900</td>\n",
       "      <td>0.07283</td>\n",
       "      <td>0.3184</td>\n",
       "      <td>0.08183</td>\n",
       "      <td>benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>9.504</td>\n",
       "      <td>12.44</td>\n",
       "      <td>60.34</td>\n",
       "      <td>273.9</td>\n",
       "      <td>0.10240</td>\n",
       "      <td>0.06492</td>\n",
       "      <td>0.02956</td>\n",
       "      <td>0.02076</td>\n",
       "      <td>0.1815</td>\n",
       "      <td>0.06905</td>\n",
       "      <td>...</td>\n",
       "      <td>15.66</td>\n",
       "      <td>65.13</td>\n",
       "      <td>314.9</td>\n",
       "      <td>0.1324</td>\n",
       "      <td>0.1148</td>\n",
       "      <td>0.08867</td>\n",
       "      <td>0.06227</td>\n",
       "      <td>0.2450</td>\n",
       "      <td>0.07773</td>\n",
       "      <td>benign</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>15.340</td>\n",
       "      <td>14.26</td>\n",
       "      <td>102.50</td>\n",
       "      <td>704.4</td>\n",
       "      <td>0.10730</td>\n",
       "      <td>0.21350</td>\n",
       "      <td>0.20770</td>\n",
       "      <td>0.09756</td>\n",
       "      <td>0.2521</td>\n",
       "      <td>0.07032</td>\n",
       "      <td>...</td>\n",
       "      <td>19.08</td>\n",
       "      <td>125.10</td>\n",
       "      <td>980.9</td>\n",
       "      <td>0.1390</td>\n",
       "      <td>0.5954</td>\n",
       "      <td>0.63050</td>\n",
       "      <td>0.23930</td>\n",
       "      <td>0.4667</td>\n",
       "      <td>0.09946</td>\n",
       "      <td>malignant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>21.160</td>\n",
       "      <td>23.04</td>\n",
       "      <td>137.20</td>\n",
       "      <td>1404.0</td>\n",
       "      <td>0.09428</td>\n",
       "      <td>0.10220</td>\n",
       "      <td>0.10970</td>\n",
       "      <td>0.08632</td>\n",
       "      <td>0.1769</td>\n",
       "      <td>0.05278</td>\n",
       "      <td>...</td>\n",
       "      <td>35.59</td>\n",
       "      <td>188.00</td>\n",
       "      <td>2615.0</td>\n",
       "      <td>0.1401</td>\n",
       "      <td>0.2600</td>\n",
       "      <td>0.31550</td>\n",
       "      <td>0.20090</td>\n",
       "      <td>0.2822</td>\n",
       "      <td>0.07526</td>\n",
       "      <td>malignant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>16.650</td>\n",
       "      <td>21.38</td>\n",
       "      <td>110.00</td>\n",
       "      <td>904.6</td>\n",
       "      <td>0.11210</td>\n",
       "      <td>0.14570</td>\n",
       "      <td>0.15250</td>\n",
       "      <td>0.09170</td>\n",
       "      <td>0.1995</td>\n",
       "      <td>0.06330</td>\n",
       "      <td>...</td>\n",
       "      <td>31.56</td>\n",
       "      <td>177.00</td>\n",
       "      <td>2215.0</td>\n",
       "      <td>0.1805</td>\n",
       "      <td>0.3578</td>\n",
       "      <td>0.46950</td>\n",
       "      <td>0.20950</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.09564</td>\n",
       "      <td>malignant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>17.140</td>\n",
       "      <td>16.40</td>\n",
       "      <td>116.00</td>\n",
       "      <td>912.7</td>\n",
       "      <td>0.11860</td>\n",
       "      <td>0.22760</td>\n",
       "      <td>0.22290</td>\n",
       "      <td>0.14010</td>\n",
       "      <td>0.3040</td>\n",
       "      <td>0.07413</td>\n",
       "      <td>...</td>\n",
       "      <td>21.40</td>\n",
       "      <td>152.40</td>\n",
       "      <td>1461.0</td>\n",
       "      <td>0.1545</td>\n",
       "      <td>0.3949</td>\n",
       "      <td>0.38530</td>\n",
       "      <td>0.25500</td>\n",
       "      <td>0.4066</td>\n",
       "      <td>0.10590</td>\n",
       "      <td>malignant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>14.580</td>\n",
       "      <td>21.53</td>\n",
       "      <td>97.41</td>\n",
       "      <td>644.8</td>\n",
       "      <td>0.10540</td>\n",
       "      <td>0.18680</td>\n",
       "      <td>0.14250</td>\n",
       "      <td>0.08783</td>\n",
       "      <td>0.2252</td>\n",
       "      <td>0.06924</td>\n",
       "      <td>...</td>\n",
       "      <td>33.21</td>\n",
       "      <td>122.40</td>\n",
       "      <td>896.9</td>\n",
       "      <td>0.1525</td>\n",
       "      <td>0.6643</td>\n",
       "      <td>0.55390</td>\n",
       "      <td>0.27010</td>\n",
       "      <td>0.4264</td>\n",
       "      <td>0.12750</td>\n",
       "      <td>malignant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>18.610</td>\n",
       "      <td>20.25</td>\n",
       "      <td>122.10</td>\n",
       "      <td>1094.0</td>\n",
       "      <td>0.09440</td>\n",
       "      <td>0.10660</td>\n",
       "      <td>0.14900</td>\n",
       "      <td>0.07731</td>\n",
       "      <td>0.1697</td>\n",
       "      <td>0.05699</td>\n",
       "      <td>...</td>\n",
       "      <td>27.26</td>\n",
       "      <td>139.90</td>\n",
       "      <td>1403.0</td>\n",
       "      <td>0.1338</td>\n",
       "      <td>0.2117</td>\n",
       "      <td>0.34460</td>\n",
       "      <td>0.14900</td>\n",
       "      <td>0.2341</td>\n",
       "      <td>0.07421</td>\n",
       "      <td>malignant</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    mean radius  mean texture  mean perimeter  mean area  mean smoothness  \\\n",
       "18       19.810         22.15          130.00     1260.0          0.09831   \n",
       "19       13.540         14.36           87.46      566.3          0.09779   \n",
       "20       13.080         15.71           85.63      520.0          0.10750   \n",
       "21        9.504         12.44           60.34      273.9          0.10240   \n",
       "22       15.340         14.26          102.50      704.4          0.10730   \n",
       "23       21.160         23.04          137.20     1404.0          0.09428   \n",
       "24       16.650         21.38          110.00      904.6          0.11210   \n",
       "25       17.140         16.40          116.00      912.7          0.11860   \n",
       "26       14.580         21.53           97.41      644.8          0.10540   \n",
       "27       18.610         20.25          122.10     1094.0          0.09440   \n",
       "\n",
       "    mean compactness  mean concavity  mean concave points  mean symmetry  \\\n",
       "18           0.10270         0.14790              0.09498         0.1582   \n",
       "19           0.08129         0.06664              0.04781         0.1885   \n",
       "20           0.12700         0.04568              0.03110         0.1967   \n",
       "21           0.06492         0.02956              0.02076         0.1815   \n",
       "22           0.21350         0.20770              0.09756         0.2521   \n",
       "23           0.10220         0.10970              0.08632         0.1769   \n",
       "24           0.14570         0.15250              0.09170         0.1995   \n",
       "25           0.22760         0.22290              0.14010         0.3040   \n",
       "26           0.18680         0.14250              0.08783         0.2252   \n",
       "27           0.10660         0.14900              0.07731         0.1697   \n",
       "\n",
       "    mean fractal dimension    ...      worst texture  worst perimeter  \\\n",
       "18                 0.05395    ...              30.88           186.80   \n",
       "19                 0.05766    ...              19.26            99.70   \n",
       "20                 0.06811    ...              20.49            96.09   \n",
       "21                 0.06905    ...              15.66            65.13   \n",
       "22                 0.07032    ...              19.08           125.10   \n",
       "23                 0.05278    ...              35.59           188.00   \n",
       "24                 0.06330    ...              31.56           177.00   \n",
       "25                 0.07413    ...              21.40           152.40   \n",
       "26                 0.06924    ...              33.21           122.40   \n",
       "27                 0.05699    ...              27.26           139.90   \n",
       "\n",
       "    worst area  worst smoothness  worst compactness  worst concavity  \\\n",
       "18      2398.0            0.1512             0.3150          0.53720   \n",
       "19       711.2            0.1440             0.1773          0.23900   \n",
       "20       630.5            0.1312             0.2776          0.18900   \n",
       "21       314.9            0.1324             0.1148          0.08867   \n",
       "22       980.9            0.1390             0.5954          0.63050   \n",
       "23      2615.0            0.1401             0.2600          0.31550   \n",
       "24      2215.0            0.1805             0.3578          0.46950   \n",
       "25      1461.0            0.1545             0.3949          0.38530   \n",
       "26       896.9            0.1525             0.6643          0.55390   \n",
       "27      1403.0            0.1338             0.2117          0.34460   \n",
       "\n",
       "    worst concave points  worst symmetry  worst fractal dimension     cancer  \n",
       "18               0.23880          0.2768                  0.07615  malignant  \n",
       "19               0.12880          0.2977                  0.07259     benign  \n",
       "20               0.07283          0.3184                  0.08183     benign  \n",
       "21               0.06227          0.2450                  0.07773     benign  \n",
       "22               0.23930          0.4667                  0.09946  malignant  \n",
       "23               0.20090          0.2822                  0.07526  malignant  \n",
       "24               0.20950          0.3613                  0.09564  malignant  \n",
       "25               0.25500          0.4066                  0.10590  malignant  \n",
       "26               0.27010          0.4264                  0.12750  malignant  \n",
       "27               0.14900          0.2341                  0.07421  malignant  \n",
       "\n",
       "[10 rows x 31 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[18:28]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "benign       357\n",
       "malignant    212\n",
       "Name: cancer, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['cancer'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Algorithm\n",
    "## Model\n",
    "In order to scale the result between 0 and 1, we use Sigmoid function.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    s = 1/(1+np.exp(-z))\n",
    "    s = s.reshape(s.shape[0],1)\n",
    "    return s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHKdJREFUeJzt3Xl8VOW9x/HPj000GlHBoiyCClVc6hLBDfcFqEKvtRa8LnVDi4jWraAUKVVU6oJ6wYpcl6tFilorrtirWJcrSnCtIL2oKEG5ibiBqCHwu388iYkxIZNkJs/Mme/79TqvnJk5Jt/R8PXwzDnPY+6OiIgkS6vYAUREJP1U7iIiCaRyFxFJIJW7iEgCqdxFRBJI5S4ikkAqdxGRBFK5i4gkkMpdRCSB2sT6wR07dvQePXrE+vEiIjlpwYIFn7h7p4aOi1buPXr0oLi4ONaPFxHJSWb2QSrHaVhGRCSBVO4iIgmkchcRSSCVu4hIAjVY7mZ2h5mVmtk/63ndzOxmM1tiZm+a2V7pjykiIo2Rypn7XcCADbw+EOhVuQ0Hbm1+LBERaY4Gy93dnwM+3cAhQ4D/8mAe0MHMtklXQBERabx0XOfeBVhW43FJ5XMfp+F7i4i0vIoKWLUKVq+Gr7+GNWvC16qt5uOa+998A+XldW/fflu9f9hhMH58Rt9COsrd6niuzoVZzWw4YeiG7t27p+FHi4jU45tv4JNP6t4+/RS+/PL72xdfVO+vWZPZbNtkfnAjHeVeAnSr8bgr8FFdB7r7NGAaQFFRkVbmFpHGq6iA5cuhpCR8rb2/fDmUlYWz7qZq1QoKC6GgADbZBDbeuHqr/bjmc+3bw0YbQbt2G95+9KP0/fuoRzrKfTYw0sxmAv2AL9xdQzIi0nTr1sG778I774SvVduSJbB0aSj4hrRtCx071r1tuSVsvnko8KqvNbeCArC6BiVyR4Plbmb3AYcAHc2sBLgCaAvg7n8CHgcGAUuANcBpmQorIglUWgqvvgr//Ce89Vb4unBhGFapz7bbQrdu0KVL3VvnzrDZZjlf0M3RYLm7+7AGXnfg3LQlEpHk+vrrUOQvvxy2V14JZ+J16dYNdt4ZdtwxbDvsELbttw9DILJB0WaFFJE8UF4eCvyZZ8L20kvhuZoKCmCvvWD33WG33WDXXcO2+eZxMieEyl1E0uvjj+HRR2H27FDoNa88MQsF3q9f2Pr2hV12gdat4+VNKJW7iDTfu+/CzJmh0F955fuv9ekDhx4aru0++GDYaqs4GfOMyl1EmqasDGbNgnvvhXnzqp9v3x6OPBIGD4ZBg8KHn9LiVO4ikjp3ePppmDoVHnmk+pLEggI47jj4+c9DsW+ySdyconIXkRR88QXcdRfceissXhyea906nJmfdFI4Sy8oiBpRvk/lLiL1Ky2FyZNhypRwWz6E68jPPhvOPLNFbqOXplG5i8gPLV8O114Lt99efTPRIYfAeeeFs/Q2qo5sp/9CIlLt889DqU+eXF3qgwfDmDGw775xs0mjqNxFJNxYNGUKXHllmDER4PjjYdy4cF265ByVu0i+mzsXRowIk3RBuBZ90qRwg5HkLC2QLZKvVqwIV7ocdlgo9t69w52lc+eq2BNAZ+4i+WjmzHC2/tln4aajsWPh4ovDXOSSCCp3kXyycmUo9VmzwuMBA8JY+/bbx80laadyF8kXf/87nHJKGI7ZdFO44YZwrXoez3meZBpzF0m69ethwgQ4+uhQ7P37wxtvwFlnqdgTTGfuIkn2ySfhQ9M5c0KRjx8fxtc1xW7iqdxFkuqNN+DYY2HZsjDN7owZcNRRsVNJC9GwjEgSPfYYHHhgKPZ+/eC111TseUblLpI0t9wSpgxYvRpOPBGefTasRyp5ReUukhTucMklMGpU+BB1/PiwkEb79rGTSQQacxdJgnXr4JxzYPr0MGPjXXfBv/977FQSkcpdJNeVl8PJJ4cbk9q3hwcfDItoSF5TuYvksvLysLTdo49CYWH42r9/7FSSBVTuIrlq7VoYOjQU+pZbhjtQ99ordirJEvpAVSQXVVSEm5Meegg6dFCxyw+o3EVyzfr1cPrpYYy9sBCeekrFLj+gchfJNZdeCvfcAwUF8MQTsM8+sRNJFlK5i+SSyZPh+uvD5Y5/+xvsv3/sRJKlVO4iuWLWLLjwwrB/551wxBFx80hWU7mL5ILnngvXsrvDtdeGD1NFNiClcjezAWa22MyWmNnoOl7vbmZzzew1M3vTzHQHhUi6LF0armUvL4eRI8MUAyINaLDczaw1MAUYCPQBhplZn1qHjQVmufuewFBgarqDiuSl1athyJAwL/uAAWHMXQtsSApSOXPvCyxx9/fcvRyYCQypdYwDhZX7mwMfpS+iSJ5yh1/9Ct58E3r3hvvu0yIbkrJUyr0LsKzG45LK52oaD5xkZiXA48B5dX0jMxtuZsVmVlxWVtaEuCJ55MorwzwxhYXw8MPhZiWRFKVS7nX9HdBrPR4G3OXuXYFBwD1m9oPv7e7T3L3I3Ys6derU+LQi+eLJJ2HcuDAEc999sNNOsRNJjkml3EuAmjP9d+WHwy5nALMA3P0loD3QMR0BRfJOSUm4MgbCwtaa4VGaIJVynw/0MrOeZtaO8IHp7FrHfAgcDmBmOxPKXeMuIo1VUQHDhoUPUI86Ci67LHYiyVENlru7VwAjgTnAIsJVMW+b2QQzG1x52EXAWWb2BnAf8Ct3rz10IyINGTcOXngBtt02TDHQSreiSNNYrA4uKiry4uLiKD9bJCs9+SQMHBgKfe5cOOig2IkkC5nZAncvaug4nRaIZIOysnDZI4RxdhW7NJPKXSQ2dxg+HP7v/+CQQ2DMmNiJJAFU7iKx3X13mOGxsDAsbK1xdkkD/RaJxLR0KYwaFfZvuQW22y5qHEkOlbtILOvWwSmnwKpVYWKwqmvbRdJA5S4Sy003wfPPQ+fO8Kc/aUIwSSuVu0gM774LY8eG/dtvh466oVvSS+Uu0tKqro75+ms48UQ45pjYiSSBVO4iLe3OO+GZZ2CrrcL87CIZoHIXaUkffwwXXRT2b74ZNDuqZIjKXaQljRwJn38eZnocNix2GkkwlbtIS3n4YfjrX2HTTeHWW3V1jGSUyl2kJaxZA+efH/avugq6d4+bRxJP5S7SEiZOhA8+gD32gBEjYqeRPKByF8m0f/0L/vjHsD91KrRpEzeP5AWVu0gmucN550F5OZx+Ouy3X+xEkidU7iKZ9OCD8NRT0KEDXHNN7DSSR1TuIpmyejX85jdhf+JEXdMuLUrlLpIpV10FJSWw995hugGRFqRyF8mE99+HG24I+1OmQOvWcfNI3lG5i2TCb38bPkQ96STo1y92GslDKneRdHv+ebj/fth4Y7j66thpJE+p3EXSaf366g9RL7kEunaNm0fylspdJJ3uvRcWLIBtt4VLL42dRvKYyl0kXb76CsaMCftXXw0FBXHzSF5TuYuky6RJ8NFHUFQUPkgViUjlLpIOy5dXzx9zww3QSn+0JC79Boqkw5VXhjVRjzsO+vePnUZE5S7SbO+9B9Onh7P1K6+MnUYEULmLNN/48VBRASefDDvvHDuNCKByF2met98Olz+2bQtXXBE7jch3Uip3MxtgZovNbImZja7nmBPMbKGZvW1mM9IbUyRLjRsX5mw/6yzo2TN2GpHvNLgkjJm1BqYARwIlwHwzm+3uC2sc0wsYAxzg7p+Z2daZCiySNYqLw4LX7dvD5ZfHTiPyPamcufcFlrj7e+5eDswEhtQ65ixgirt/BuDupemNKZKFxo4NX0eODHekimSRVMq9C7CsxuOSyudq6g30NrMXzWyemQ2o6xuZ2XAzKzaz4rKysqYlFskGzz8Pc+bAZpuFGSBFskwq5W51POe1HrcBegGHAMOA6WbW4Qf/kPs0dy9y96JOWpVGcpU7XHZZ2L/wQujYMW4ekTqkUu4lQLcaj7sCH9VxzMPuvtbd3wcWE8peJHnmzIEXXoAttwzlLpKFUin3+UAvM+tpZu2AocDsWsf8DTgUwMw6EoZp3ktnUJGs4F491j56NBQWxs0jUo8Gy93dK4CRwBxgETDL3d82swlmNrjysDnASjNbCMwFLnH3lZkKLRLNQw+FKX07d4Zzz42dRqRe5l57+LxlFBUVeXFxcZSfLdIk69bBbrvBokVhXdQRI2InkjxkZgvcvaih43SHqkiqZswIxd6jB5x5Zuw0IhukchdJRXl59fQC48dDu3ZR44g0ROUukoo77oD334eddtJCHJITVO4iDfn6a/jDH8L+hAnQunXcPCIpULmLNGTq1LB83p57ws9/HjuNSEpU7iIb8uWXYbFrCAtxaPk8yRH6TRXZkMmTYeVKOOAAGDgwdhqRlKncRerz6adw/fVh/6qrwOqaZkkkO6ncReozaVIYljnySDj44NhpRBpF5S5Sl48/hptvDvtXXRU3i0gTqNxF6jJxYrgE8mc/g332iZ1GpNFU7iK1LV0Kt90Wxtirrm8XyTEqd5HaJkyAtWvhxBNh111jpxFpEpW7SE2LF8Pdd4e7UMePj51GpMlU7iI1jRsH69fDGWfAjjvGTiPSZCp3kSqvvw6zZsFGG8Hvfhc7jUizqNxFqlQV+ogR0LVr3CwizaRyFwF46SV49FEoKAhro4rkOJW7iDtcdlnYv+AC2HrruHlE0kDlLvL00/Dss9ChA1x8cew0Immhcpf85g6XXx72L700FLxIAqjcJb898gi88koYihk1KnYakbRRuUv+Wr8exo4N+5dfHj5MFUkIlbvkr7/8Bd56C7p1g7PPjp1GJK1U7pKf1q4Nd6MCXHFFuHFJJEFU7pKf7r4bliyBXr3g1FNjpxFJO5W75J9vvoHf/z7sT5gAbdrEzSOSASp3yT9Tp0JJCeyxB5xwQuw0Ihmhcpf88uWXYZUlCMvntdIfAUkm/WZLfrn+eli5Eg48EAYOjJ1GJGNSKnczG2Bmi81siZnVO6uSmR1vZm5mRemLKJImZWVwww1h/+qrwzJ6IgnVYLmbWWtgCjAQ6AMMM7M+dRy3GTAKeDndIUXSYuJEWL0aBg0KZ+4iCZbKmXtfYIm7v+fu5cBMYEgdx/0BmAR8k8Z8Iunx4Yfhg1SoHnMXSbBUyr0LsKzG45LK575jZnsC3dz90TRmE0mf3/8eysth2DD4yU9ipxHJuFTKva6BSf/uRbNWwI3ARQ1+I7PhZlZsZsVlZWWppxRpjnfegbvuCtezT5gQO41Ii0il3EuAbjUedwU+qvF4M2BX4FkzWwrsC8yu60NVd5/m7kXuXtSpU6empxZpjLFjtei15J1Uyn0+0MvMeppZO2AoMLvqRXf/wt07unsPd+8BzAMGu3txRhKLNMb8+fDgg9C+vRa9lrzSYLm7ewUwEpgDLAJmufvbZjbBzAZnOqBIk7nDJZeE/VGjoEuXDR8vkiDm7g0flQFFRUVeXKyTe8mg2bNhyBDYaqswSZhWWZIEMLMF7t7gvUS6Q1WSae3asGwehKl9VeySZ1TukkzTp8PixeED1HPOiZ1GpMWp3CV5vvwSxo8P+9dcA+3aRY0jEoPKXZJn0iQoLYX994fjjoudRiQKlbskS0lJmPkR4LrrNDmY5C2VuyTL734XVlr6xS9gv/1ipxGJRuUuyfHqq2Ft1LZtw5S+InlM5S7J4A7nnVf9dYcdYicSiUrlLskwYwb8z//Aj34EV1wRO41IdCp3yX2rVlVPM3DNNVBYGDePSBZQuUvumzgRPv4Y+vaFU06JnUYkK6jcJbf97/9Wr4t6yy3QSr/SIqByl1z3m9+EFZZOOy2cuYsIoHKXXPbYY2ErLNSljyK1qNwlN61ZAyNHhv0rrghXyYjId1TukpsmTIClS8Ni16NGxU4jknVU7pJ73norzB9jBrfdFha+FpHvUblLblm/Hs4+Gyoq4Ne/hn79YicSyUoqd8ktt98OL70EnTuH69tFpE4qd8kdK1bA6NFh/6abYPPN4+YRyWIqd8kN7mG5vM8/hwEDwpS+IlIvlbvkhvvug4cfhs02Cx+iahEOkQ1SuUv2W7EiTOMLYaqB7t3j5hHJASp3yW5VwzGffgpHHw1nnBE7kUhOULlLdpsxIwzHFBaGK2U0HCOSEpW7ZK/ly78/HNOtW9w8IjlE5S7Zad06OPlk+OwzGDgQTj89diKRnKJyl+x03XUwdy5svTXceaeGY0QaSeUu2Wf+fBg7NuzfeadmfBRpApW7ZJdVq+DEE8PcMeefD4MGxU4kkpNU7pI93GHECFiyBHbfPSx2LSJNklK5m9kAM1tsZkvMbHQdr19oZgvN7E0ze9rMtkt/VEm8W2+Fe++FTTYJd6S2bx87kUjOarDczaw1MAUYCPQBhplZn1qHvQYUufvuwAPApHQHlYSbNw8uuCDsT58OfWr/iolIY6Ry5t4XWOLu77l7OTATGFLzAHef6+5rKh/OA7qmN6YkWllZmAhs7dpwXfuwYbETieS8VMq9C7CsxuOSyufqcwbwRHNCSR6pqAhlXlIC++0XLoEUkWZLZX2yui4w9joPNDsJKAIOruf14cBwgO6a/EkALroInn46XM9+//3Qrl3sRCKJkMqZewlQ877vrsBHtQ8ysyOAy4HB7v5tXd/I3ae5e5G7F3Xq1KkpeSVJpk6Fm2+Gtm3hgQegy4b+QigijZFKuc8HeplZTzNrBwwFZtc8wMz2BG4jFHtp+mNK4jz1FIwaFfanT4f+/ePmEUmYBsvd3SuAkcAcYBEwy93fNrMJZja48rA/ApsC95vZ62Y2u55vJwILF8IJJ4T5Y8aMgVNOiZ1IJHFSGXPH3R8HHq/13Lga+0ekOZck1YcfhnnZv/gCjjsOrrwydiKRRNIdqtJyPvkEjjoqXBlz4IFwzz3QSr+CIpmgP1nSMlatCvPELF4Mu+0GjzwS7kQVkYxQuUvmrVkDP/tZmO2xZ0+YMwc6dIidSiTRVO6SWWvWwLHHwjPPQOfO4SqZbbaJnUok8VL6QFWkSdasgWOOCYtudO4cvu64Y+xUInlBZ+6SGatW/bDYd9opdiqRvKEzd0m/0tLw4emCBaHYn30Wfvzj2KlE8orO3CW93n8fDjggFPsOO8ALL6jYRSJQuUv6vPoq7L9/WElpjz3gxRdDwYtIi1O5S3rMmhVuTFqxAg49FP7xDy1sLRKRyl2aZ/16GDcOfvlL+PprOP10eOIJKCyMnUwkr+kDVWm6lSvhtNPC3aatWsH118P554PVtQSAiLQklbs0zYsvhhWUli0Ld5vOnBkmBBORrKBhGWmcdevgmmvg4INDse+7L7z+uopdJMuo3CV1ixfDQQeFOdjXrYNLLoHnnoPttoudTERq0bCMNGzdOpg8GcaOhW++CXPDTJ8eblQSkaykcpcNe+UVOPdcKC4Oj089FW68EbbYIm4uEdkgDctI3UpL4YwzoF+/UOxdusCjj8Jdd6nYRXKAyl2+76uv4OqroXdvuOMOaNsWRo+Gd96Bn/40djoRSZGGZST49luYNi2saVpaGp4bODCMtffuHTebiDSayj3frVoVPhy98cZwaSNA374wcSIcfnjcbCLSZCr3fLViBdx8M9x6K3z+eXhul13gqqtg8GDdZSqS41Tu+WT9+rBoxrRp8NBDsHZteP7AA8M168ccE6YREJGcp3LPBx98APfdF4Zf3n03PNeqFfzbv4VS32+/uPlEJO1U7kn10Udw//3wl7/ASy9VP9+tG5x5Zpi9sWvXePlEJKNU7kmxfn2Y4+Wxx+Dxx+Hll8E9vLbJJnDssXDyyTBgALRuHTeriGScyj2XffhhWBRj7twwh/qKFdWvtWsXpgcYOjSMpRcUxMspIi1O5Z4r1q6FhQvDdADPPx8m7Prgg+8f06VLuNFo0KBwGeOmm8bJKiLRqdyz0VdfhRkYX3stLDS9YAG88Ua40aimDh2gf/+wHX007LabLmEUEUDlHs+6dbB8OSxdGop80aLqrfYZeZUddoC99w6XLh50EOy6q8bPRaROKvdMWLcOysrCGHjVtmxZKPKq7cMPoaKi7n++bVvo1Sucie+9d9j22iucqYuIpCClcjezAcBNQGtgurtfU+v1jYD/AvYGVgK/dPel6Y0aybffwmeffX/79NMfPldaWl3kpaXh6pWGbLNNWOiiVy/YeefqbfvtQ8GLiDRRg+VuZq2BKcCRQAkw38xmu/vCGoedAXzm7jua2VDgWuCXmQjMqlVhTPrbb8NWXl6939Djqv01a2D16vB9GvpadRdnY3XqBJ07V2/bbgs9e0KPHmHr3h023jid/2ZERL6Typl7X2CJu78HYGYzgSFAzXIfAoyv3H8A+A8zM/eqC63T6Pjj4amn0v5t69WmTZi/fIstYMstq/drP1ezzLfeWmfeIhJVKuXeBVhW43EJ0K++Y9y9wsy+ALYCPql5kJkNB4YDdO/evWmJt9oqlOdGG4WtXbvq/VSf22STcJlgQUHYqvbr+tquXdNyiohElEq513VtXe0z8lSOwd2nAdMAioqKmnZWP2NGk/4xEZF8ksoUgCVAtxqPuwIf1XeMmbUBNgc+TUdAERFpvFTKfT7Qy8x6mlk7YCgwu9Yxs4FTK/ePB57JyHi7iIikpMFhmcox9JHAHMKlkHe4+9tmNgEodvfZwH8C95jZEsIZ+9BMhhYRkQ1L6Tp3d38ceLzWc+Nq7H8D/CK90UREpKm07I6ISAKp3EVEEkjlLiKSQCp3EZEEslhXLJpZGVDP3LYN6kitu19zmN5LdkrKe0nK+wC9lyrbuXunhg6KVu7NYWbF7l4UO0c66L1kp6S8l6S8D9B7aSwNy4iIJJDKXUQkgXK13KfFDpBGei/ZKSnvJSnvA/ReGiUnx9xFRGTDcvXMXURENiCny93MzjOzxWb2tplNip2nuczsYjNzM+sYO0tTmdkfzewdM3vTzB4ys5xa1dvMBlT+Ti0xs9Gx8zSVmXUzs7lmtqjyz8f5sTM1h5m1NrPXzOzR2Fmaw8w6mNkDlX9GFpnZfpn6WTlb7mZ2KGF5v93dfRfgusiRmsXMuhHWqf0wdpZm+juwq7vvDvwLGBM5T8pqrBc8EOgDDDOzPnFTNVkFcJG77wzsC5ybw+8F4HxgUewQaXAT8KS77wT8hAy+p5wtd+DXwDXu/i2Au5dGztNcNwKXUscKVrnE3Z9y94rKh/MIi7vkiu/WC3b3cqBqveCc4+4fu/urlfurCCXSJW6qpjGzrsBPgemxszSHmRUCBxGmSMfdy93980z9vFwu995AfzN72cz+YWb7xA7UVGY2GFju7m/EzpJmpwNPxA7RCHWtF5yThViTmfUA9gRejpukySYTTnzWxw7STNsDZcCdlUNM082sIFM/LKX53GMxs/8GOtfx0uWE7FsQ/sq5DzDLzLbP1hWgGngvlwFHtWyiptvQe3H3hyuPuZwwNPDnlszWTCmtBZxLzGxT4EHgAnf/MnaexjKzY4BSd19gZofEztNMbYC9gPPc/WUzuwkYDfwuUz8sa7n7EfW9Zma/Bv5aWeavmNl6wnwNZS2VrzHqey9mthvQE3jDzCAMY7xqZn3dfUULRkzZhv67AJjZqcAxwOHZ+j/beqSyXnDOMLO2hGL/s7v/NXaeJjoAGGxmg4D2QKGZ3evuJ0XO1RQlQIm7V/0N6gFCuWdELg/L/A04DMDMegPtyMFJhdz9LXff2t17uHsPwi/AXtla7A0xswHAb4HB7r4mdp5GSmW94Jxg4UzhP4FF7n5D7DxN5e5j3L1r5Z+NoYT1mXOx2Kn8M73MzH5c+dThwMJM/bysPnNvwB3AHWb2T6AcODXHzhKT6j+AjYC/V/5NZJ67nxM3UmrqWy84cqymOgA4GXjLzF6vfO6yyiUzJZ7zgD9Xnjy8B5yWqR+kO1RFRBIol4dlRESkHip3EZEEUrmLiCSQyl1EJIFU7iIiCaRyFxFJIJW7iEgCqdxFRBLo/wFHtlfjk5n1iQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def draw_sigmoid():\n",
    "    x = np.arange(-6, 6, .01)\n",
    "    y = sigmoid(x)\n",
    "\n",
    "    plt.plot(x, y, color='red', lw=2)\n",
    "    plt.show()\n",
    "\n",
    "draw_sigmoid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "when we replace z with $\\theta x$, we get the model function.\n",
    "\n",
    "$\\hat{y}=\\frac{1}{1 + e^{- \\theta x}}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(theta, X):\n",
    "    z = np.sum(theta.T * X, axis=1)\n",
    "    return sigmoid(z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loss Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cross_entropy\n",
    "def cross_entropy(y, y_hat):\n",
    "    n_samples = y.shape[0]\n",
    "    return sum(-y*np.log(y_hat)-(1-y)*np.log(1-y_hat))/n_samples\n",
    "\n",
    "def cost_function(theta, X, y):\n",
    "    y_hat = model(theta, X)\n",
    "    return cross_entropy(y, y_hat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizer\n",
    "\n",
    "Let us name the lost function as **J**\n",
    "\n",
    "$J= - y * log(\\hat{y}) - (1-y) * log(1-\\hat{y})$\n",
    "\n",
    "So the devivative of J will be:\n",
    "\n",
    "$\\frac{\\partial J}{\\partial \\theta} = - \\frac{\\partial (y * log(\\hat{y}) + (1-y) * log(1-\\hat{y}))}{\\partial \\theta}$\n",
    "\n",
    "$ = - \\frac{\\partial (y * log(\\hat{y}))}{\\partial \\theta} - \\frac{\\partial  ((1-y) * log(1-\\hat{y}))}{\\partial \\theta}$\n",
    "\n",
    "$ = - y * \\frac{\\partial log(\\hat{y})}{\\partial \\theta} - (1-y) * \\frac{log(1-\\hat{y})}{\\partial \\theta}$\n",
    "\n",
    "$ = - \\frac{y}{\\hat{y}} *\\frac{\\partial \\hat{y}}{\\partial \\theta} - \\frac{1-y}{1-\\hat{y}} *\\frac{\\partial (1-\\hat{y})}{\\partial \\theta}$\n",
    "\n",
    "$ = - \\frac{y}{\\hat{y}} *\\frac{\\partial \\hat{y}}{\\partial \\theta} + \\frac{1-y}{1-\\hat{y}} *\\frac{\\partial (\\hat{y})}{\\partial \\theta}$\n",
    "\n",
    "$ = (- \\frac{y}{\\hat{y}} + \\frac{1-y}{1-\\hat{y}} ) * \\frac{\\partial (\\hat{y})}{\\partial \\theta}$\n",
    "\n",
    "$ = (- \\frac{y}{\\hat{y}} + \\frac{1-y}{1-\\hat{y}} ) * \\frac{\\partial (\\frac{1}{1 + e^{- \\theta x}})} {\\partial a}$\n",
    "\n",
    "$ = (- \\frac{y}{\\hat{y}} + \\frac{1-y}{1-\\hat{y}} ) * (- \\hat{y} ^ 2) \\frac{\\partial (1 + e^{- \\theta x})} {\\partial \\theta}$\n",
    "\n",
    "$ = (- \\frac{y}{\\hat{y}} + \\frac{1-y}{1-\\hat{y}} ) * (- \\hat{y} ^ 2) \\frac{\\partial (e^{- \\theta x})} {\\partial \\theta}$\n",
    "\n",
    "$ = (- \\frac{y}{\\hat{y}} + \\frac{1-y}{1-\\hat{y}} ) * (- \\hat{y} ^ 2) * e^{- \\theta x} \\frac{\\partial (- \\theta x)} {\\partial \\theta}$\n",
    "\n",
    "$ = (- \\frac{y}{\\hat{y}} + \\frac{1-y}{1-\\hat{y}} ) * \\hat{y} ^ 2 * e^{- \\theta x} * x $\n",
    "\n",
    "∵  $\\hat{y}=\\frac{1}{1 + e^{- \\theta x}}$ \n",
    "\n",
    "∴  $1 + e^{- \\theta x}=\\frac{1}{\\hat{y}}$ \n",
    "\n",
    "∴  $e^{- \\theta x}=\\frac{1}{\\hat{y}} - 1$ \n",
    "\n",
    "∴  $e^{- \\theta x}=\\frac{1-\\hat{y}}{\\hat{y}}$ \n",
    "\n",
    "$\\frac{\\partial J}{\\partial \\theta} = (- \\frac{y}{\\hat{y}} + \\frac{1-y}{1-\\hat{y}} ) * \\hat{y} ^ 2 * \\frac{1-\\hat{y}}{\\hat{y}} * x$\n",
    "\n",
    "$\\frac{\\partial J}{\\partial \\theta} = (- \\frac{y}{\\hat{y}} + \\frac{1-y}{1-\\hat{y}} ) * \\hat{y} * (1-\\hat{y}) * x$\n",
    "\n",
    "$\\frac{\\partial J}{\\partial \\theta} = (- y * (1-\\hat{y}) + (1-y) * \\hat{y} ) * x$\n",
    "\n",
    "$\\frac{\\partial J}{\\partial \\theta} = (- y + y *\\hat{y} + \\hat{y}-y * \\hat{y} ) * x$\n",
    "\n",
    "$\\frac{\\partial J}{\\partial \\theta} = (\\hat{y}-y ) * x$\n",
    "\n",
    "Thus the optimizer will be:\n",
    "\n",
    "$\\theta = \\theta - \\alpha * \\frac{\\partial J}{\\partial \\theta}$\n",
    "\n",
    "$\\theta = \\theta - \\alpha * (\\hat{y}-y ) * x$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us write the code of it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize(theta,X,y):\n",
    "    n = X.shape[0]\n",
    "    alpha = 1e-1\n",
    "    y_hat = model(theta,X)\n",
    "    dtheta = (1.0/n) * ((y_hat-y)*X)\n",
    "    dtheta = np.sum(dtheta, axis=0)\n",
    "    dtheta=dtheta.reshape((31,1))\n",
    "    theta = theta - alpha * dtheta\n",
    "    return theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation\n",
    "We evaluate the performance of classification algorithms by accuracy. You can also use ROC, recall, precision, F1 score and so on. But here I just use accuracy. Three functions are given. The first calcuate $\\hat{y}$, which is a number between 0 and 1. The second function use the result of the first function and "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_proba(theta, X):\n",
    "    y_hat=model(theta, X)\n",
    "    return y_hat\n",
    "\n",
    "def predict(X, theta):\n",
    "    y_hat=predict_proba(theta,X)\n",
    "    y_hard=(y_hat > 0.5) * 1\n",
    "    return y_hard\n",
    "\n",
    "def accuracy(theta, X, y):\n",
    "    y_hard=predict(X, theta)\n",
    "    count_right=sum(y_hard == y)\n",
    "    return count_right*1.0/len(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def iterate(theta,X,y,times):\n",
    "    costs = []\n",
    "    accs = []\n",
    "    for i in range(times):\n",
    "        theta = optimize(theta,X,y)\n",
    "        costs.append(cost_function(theta, X, y))\n",
    "        accs.append(accuracy(theta, X, y))\n",
    "\n",
    "    return theta, costs, accs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Data\n",
    "Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = dataset.data\n",
    "y = dataset.target\n",
    "n_features = X.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "std=X.std(axis=0)\n",
    "mean=X.mean(axis=0)\n",
    "X_norm = (X - mean) / std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Append ones at the beginning of each row of matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_ones(X):\n",
    "    ones=np.ones((X.shape[0],1))\n",
    "    X_with_ones=np.hstack((ones, X))\n",
    "    return X_with_ones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_with_ones = add_ones(X_norm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split Train and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_with_ones, y, test_size = 0.3, random_state=12345)\n",
    "y_train=y_train.reshape((y_train.shape[0],1))\n",
    "y_test=y_test.reshape((y_test.shape[0],1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Apply Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta = np.ones((n_features+1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: RuntimeWarning: divide by zero encountered in log\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "theta, costs, accs = iterate(theta, X_train, y_train, 1500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1e4af03b438>]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFABJREFUeJzt3X2MXFd9xvHnuTO7XnvtxHaySQ1O4qRFEWkRhK4Caaq+pCGEFKWqxB9BtKUtyGqrStBSUaKolfivQIVoKwREQEEQApS3ooiXRpCAEBBYQxIc8mZCgs2bNzhO7Dixd3d+/eOe2Z1dz8ydXfvunIHvRxnNnTtn7vx8N/Ps2XPPveOIEABgdBTDLgAAsDoENwCMGIIbAEYMwQ0AI4bgBoARQ3ADwIghuAFgxBDcADBiCG4AGDHNOjZ69tlnx65du+rYNAD8UtqzZ89jETE1SNtagnvXrl2amZmpY9MA8EvJ9qODtmWoBABGDMENACOG4AaAEUNwA8CIIbgBYMQQ3AAwYghuABgxWQX3f33pIX3lwdlhlwEAWcsquN/1lR/oaw8R3ADQT1bBXdhq8d3FANBXVsFtSy2+dR4A+soquAtb5DYA9JdZcNPjBoAqmQW3CW4AqJBVcJuDkwBQKavgLiwFPW4A6Cuz4LZarWFXAQB5yyy4OTgJAFWyCm7GuAGgWlbBXRSMcQNAlbyCm+mAAFBpoG95t/2IpCOSFiTNR8R0HcVwrRIAqDZQcCd/GBGP1VaJuFYJAAwiu6ESchsA+hs0uEPS/9neY3t3bcXQ4waASoMOlVwRET+xfY6k22zfHxFf7WyQAn23JJ1//vlrKoaDkwBQbaAed0T8JN0flPRpSZd1aXNTRExHxPTU1NSaimEeNwBUqwxu25O2t7SXJV0taW8txXCtEgCoNMhQybmSPm273f4jEfGFOophOiAAVKsM7oh4WNLz16EWDk4CwACymg7IGDcAVMsquBnjBoBqmQU30wEBoEp+wc0XKQBAX1kFN9cqAYBqWQU31yoBgGp5BXchLZDcANBXXsHNwUkAqJRhcA+7CgDIW2bBzTxuAKiSWXAzVAIAVbIKbjOPGwAqZRXcXGQKAKplFtzM4waAKnkFd0GPGwCqZBXc5uAkAFTKKrgZKgGAapkFN0MlAFAls+DmzEkAqJJVcHNZVwCollVwM8YNANUyC2563ABQJbPgZjogAFTJKrjNwUkAqJRVcHNZVwColllw0+MGgCqZBTcHJwGgSlbBXV6Pm+AGgH4GDm7bDdvftX1rbcUwjxsAKq2mx/06SffVVYjEUAkADGKg4La9U9IfS3pvrcUUHJwEgCqD9rjfIemNkmr9RkiuVQIA1SqD2/bLJR2MiD0V7XbbnrE9Mzs7u7ZiGOMGgEqD9LivkHSd7UckfVTSlbY/vLJRRNwUEdMRMT01NbW2YuhxA0ClyuCOiBsiYmdE7JJ0vaQvR8Sf1VIM1yoBgEr5zeMOTnsHgH6aq2kcEXdIuqOWSlQOlZTvUx6oBACcLKsedyOlNcMlANBbVsFdFO3gHnIhAJCxrIK7PTxCjxsAessquIuU3OQ2APSWWXCX9/S4AaC3zIKbg5MAUCWr4LY5OAkAVbIK7qV53CQ3APSSWXDT4waAKpkFd3nPGDcA9JZVcJuDkwBQKavgZh43AFTLLLjLe3rcANBbZsHNwUkAqJJVcC9eq4TkBoCesgpuxrgBoFpewZ2qYYwbAHrLK7iZDggAlbIKbq5VAgDVsgpurlUCANUyC2563ABQJbPgLu8Z4waA3rIKbq5VAgDVsgpu5nEDQLXMgru8p8cNAL1lFtwcnASAKlkFt+lxA0ClrIJ7aYyb4AaAXiqD2/aE7W/Zvtv2vbbfXFsxKbgXWnW9AwCMvuYAbY5LujIijtoek/Q125+PiG+e7mI4OAkA1SqDO8pxi6Pp4Vi61ZKsRcE8bgCoMtAYt+2G7bskHZR0W0Tc2aXNbtsztmdmZ2fXVgzzuAGg0kDBHRELEfECSTslXWb7t7q0uSkipiNiempqam3FMFQCAJVWNaskIg5LukPSNXUUw2VdAaDaILNKpmxvTcsbJV0l6f5aiqHHDQCVBplVskPSB203VAb9xyPi1jqKYR43AFQbZFbJPZIuXYdalk55Zx43APSU1ZmTnPIOANWyCm4uMgUA1fIK7lQNY9wA0FtewU2PGwAqZRbc5T1j3ADQW1bBzXdOAkC1rIK7ILgBoFJmwV3eM48bAHrLKrgbKbkX6HEDQE95BjfTSgCgp7yC2wQ3AFTJKrj5BhwAqJZVcNPjBoBqWQV3wRg3AFTKKrgbDJUAQKW8gntxqGTIhQBAxvIKbnrcAFApy+CeXyC4AaCXrIK7fco7Z04CQG9ZBbdtFZZazCoBgJ6yCm6pHC6hxw0AvWUX3IVNjxsA+sguuBuFOQEHAPrIL7jNUAkA9JNfcDfocQNAP/kFtwluAOgnu+AuCnPmJAD0URncts+zfbvt+2zfa/t1dRZEjxsA+msO0GZe0hsi4ju2t0jaY/u2iPh+HQWVs0rq2DIA/HKo7HFHxE8j4jtp+Yik+yQ9u7aCCi4yBQD9rGqM2/YuSZdKurOOYiSGSgCgysDBbXuzpE9Ken1EPNnl+d22Z2zPzM7Orr0gTnkHgL4GCm7bYypD++aI+FS3NhFxU0RMR8T01NTUmgtqFtYCl3UFgJ4GmVViSe+TdF9EvL32gjhzEgD6GqTHfYWkP5d0pe270u3augpqFFxkCgD6qZwOGBFfk+R1qEUSl3UFgCr5nTnJrBIA6Cu74G5wyjsA9JVfcNPjBoC+8gvuwmpxyjsA9JRlcM+T3ADQU3bBXZ45OewqACBf2QV3w2IeNwD0kV9w82XBANBXdsFdmOmAANBPdsFNjxsA+ssuuLmsKwD0l11wN+lxA0Bf2QU3Z04CQH/ZBXfBZV0BoK/sgrvBFykAQF/ZBXdRWAuc8Q4APWUX3I1CzOMGgD7yC24OTgJAX9kFNwcnAaC/7IK7WVjzBDcA9JRdcHPmJAD0l11wjxUFY9wA0Ed+wd0og5vwBoDu8gvupiVJc0zmBoCusgvu8UZZ0gmCGwC6yi+4m2VJc/MENwB0k11wj6Ue9xzfGAwAXWUb3CfocQNAV5XBbfv9tg/a3rseBY01yoOTjHEDQHeD9Lg/IOmamutYNL44VEJwA0A3lcEdEV+VdGgdapHUcXCS4AaArk7bGLft3bZnbM/Mzs6ueTtj9LgBoK/TFtwRcVNETEfE9NTU1Jq30w7u4xycBICusptVMr545iTTAQGgm+yCe3GohB43AHQ1yHTAWyR9Q9LFtg/Yfk2dBXFwEgD6a1Y1iIhXrkchbWNcqwQA+spuqGScMycBoK/sgnvDWFnSMwQ3AHSVXXBPjpejN8eOzw+5EgDIU3bBvXGsIUl66sTCkCsBgDxlF9xFYW0ca+jpE/S4AaCb7IJbkiY3NOhxA0APWQb3pvGmnia4AaCrTIO7oac4OAkAXWUb3E/P0eMGgG6yDO7JDU0dpccNAF1lGdxnTIzpyafnhl0GAGQpy+DeNjmmx48R3ADQTZbBvX3TuA4fO6GFFtfkBoCVsgzubZPjaoX0BMMlAHCSLIN7++S4JOnQU8eHXAkA5CfL4J7askGS9PMnCW4AWCnL4D5/+yZJ0v5Dx4ZcCQDkJ8vg3nHmRjULa//jBDcArJRlcDcK64KzNunBnx8ddikAkJ0sg1uSnr9zq+7af1gRTAkEgE7ZBveLLtqu2SPH9b0fPzHsUgAgK9kG9zW/uUPjjUK3fGv/sEsBgKxkG9xnbhrT9Zedp499+0f6+r7Hhl0OAGQj2+CWpDdcfbF+45zNes0HZ/Shbz6q+QW++R0Asg7uMzeO6cOvfZF++4Jt+pfP7NXvv+0OvfUL92vPo4/rxDwhDuBXk+uYtTE9PR0zMzOnbXutVujL9x/UB77+iL7x8C+00AqNNwo991ln6OJzN+uCsya166xJnbd9o87ZMqGzNo9rrJH17yQAWMb2noiYHqRts+5iToeisK665Fxddcm5OnzshL7+g1/o7v2Hddf+w7r9gVnNHjlw0mu2bhrT1OYN2j45ri0TYzpjoqktE01tmRjTlommNk80tXlDUxvHGtow1tBEs9DEWCPdyuUNzaV720P4lwPAyQYKbtvXSPoPSQ1J742If6u1qj62bhrXtc/boWuft2Nx3VPH5/WjQ8e0/9AxPXb0hGaPHNdjR49r9shxHXrqhA48fkxHnpnX0ePzOvLMnNZytdhmYTUb1lhRqNmwmo3i5HVFobGG1SjK58c61hUub43CssuTjNrrivTYthqF1HB7uXyuSG0bXvG4va3FZauRnrckubwvXLZrLystO22vXJXapNfY6XVpWWq3TdssyteoY7vLt7O03Pn+y98jbTO1U+d2Ore1WENZR3u5vaq9PXW069zmsvVpwSvattt0vM2y9UuvWXr90muWGpxSbV3q6FovnYhfeZXBbbsh6Z2SXiLpgKRv2/5sRHy/7uIGNbmhqefuOEPP3XFGZduI0LETCynI5/TMXEvPzC3ombmWjs8vLD2eX75ufqGl+VZobqGl+YXQfKt9v3zd3EJoIa0rXzevuYVQK9q3cuinFaGFCLVaWnxuYdlyKEJaaC1/LdcoR6dV/VI56TX9fwl2vqb93Mr37XztSW261Lnymd7b6Vy/uvdd9k5dfkGuZZvdyu/WdvumcX38by7vWsvpNEiP+zJJ+yLiYUmy/VFJfyIpm+BeDdua3NDU5IampIlhl7MmkYK9FVoM+VbHL4GFKEM/FEr/qbW4rvzFUW6nbNOKcpvRXtex3OrYVvnc0nKro50Wt7P0+lZ0f/9Ysc1+799uU76DTjqTdvHfuViHFt+nva8WX7G4neXb7LadpddEx3svbfOkdYPWtmx5qY7On+0p1dalXefr1aXeQWvrrHFxecW/d2l99Fjfvb16tT+FbXa277E44L/l5PY9SteWifUZfR7kXZ4tqfMsmAOSXlRPORiEXQ7RAPjVNMjUi24JcdLvYtu7bc/YnpmdnT31ygAAXQ0S3AckndfxeKekn6xsFBE3RcR0RExPTU2drvoAACsMEtzflvQc2xfaHpd0vaTP1lsWAKCXyjHuiJi3/feSvqhyOuD7I+Le2isDAHQ10CHQiPicpM/VXAsAYACcFw4AI4bgBoARQ3ADwIip5eqAtmclPbrKl50tKfdvTKDGU5d7fRI1ni7UuDoXRMRAc6lrCe61sD0z6CUNh4UaT13u9UnUeLpQY30YKgGAEUNwA8CIySm4bxp2AQOgxlOXe30SNZ4u1FiTbMa4AQCDyanHDQAYwNCD2/Y1th+wvc/2m4ZYx3m2b7d9n+17bb8urd9u+zbbD6X7bWm9bf9nqvse2y9cx1obtr9r+9b0+ELbd6YaP5YuBibbG9Ljfen5XetU31bbn7B9f9qfl+e2H23/Q/o577V9i+2JYe9H2++3fdD23o51q95vtl+d2j9k+9XrUOPb0s/6Htuftr2147kbUo0P2H5px/paPvfd6ut47p9sh+2z0+Oh7MPTovzGjeHcVF606geSLpI0LuluSZcMqZYdkl6YlrdIelDSJZLeKulNaf2bJL0lLV8r6fMqr1f+Ykl3rmOt/yjpI5JuTY8/Lun6tPxuSX+blv9O0rvT8vWSPrZO9X1Q0mvT8rikrTntR5VfDvJDSRs79t9fDns/Svo9SS+UtLdj3ar2m6Ttkh5O99vS8raaa7xaUjMtv6WjxkvSZ3qDpAvTZ71R5+e+W31p/XkqL5T3qKSzh7kPT8u/c6hvLl0u6Ysdj2+QdMOwd0qq5X9Vfs/mA5J2pHU7JD2Qlt8j6ZUd7Rfb1VzXTklfknSlpFvT/3SPdXxwFvdp+h/18rTcTO1cc31npFD0ivXZ7EctfavT9rRfbpX00hz2o6RdK0JxVftN0islvadj/bJ2ddS44rk/lXRzWl72eW7vx7o/993qk/QJSc+X9IiWgnto+/BUb8MeKun2tWjPHlIti9KfwpdKulPSuRHxU0lK9+ekZsOq/R2S3iiplR6fJelwRMx3qWOxxvT8E6l9nS6SNCvpv9NwznttTyqj/RgRP5b075J+JOmnKvfLHuW1H9tWu9+G/Zn6a5W9WPWpZV1rtH2dpB9HxN0rnsqivrUYdnAP9LVo68n2ZkmflPT6iHiyX9Mu62qt3fbLJR2MiD0D1jGM/dtU+afquyLiUklPqfwTv5dh7MdtKr/w+kJJz5I0KellferI7v9T9a5paLXavlHSvKSb26t61LJuNdreJOlGSf/a7ekedeT4815m2ME90NeirRfbYypD++aI+FRa/XPbO9LzOyQdTOuHUfsVkq6z/Yikj6ocLnmHpK2229dW76xjscb0/JmSDtVc4wFJByLizvT4EyqDPKf9eJWkH0bEbETMSfqUpN9RXvuxbbX7bSifqXQA7+WSXhVpfCGTGn9d5S/ou9PnZqek79j+tUzqW5NhB3c2X4tm25LeJ+m+iHh7x1OfldQ+qvxqlWPf7fV/kY5Mv1jSE+0/aesSETdExM6I2KVyX305Il4l6XZJr+hRY7v2V6T2tfYcIuJnkvbbvjit+iNJ31dG+1HlEMmLbW9KP/d2jdnsxw6r3W9flHS17W3pL4ur07ra2L5G0j9Lui4ijq2o/fo0K+dCSc+R9C2t4+c+Ir4XEedExK70uTmgchLCz5TRPly1YQ+yqzyy+6DKo8w3DrGO31X559A9ku5Kt2tVjmV+SdJD6X57am9J70x1f0/S9DrX+wdamlVykcoPxD5J/yNpQ1o/kR7vS89ftE61vUDSTNqXn1F5ZD6r/SjpzZLul7RX0odUznwY6n6UdIvKMfc5lQHzmrXsN5XjzPvS7a/WocZ9KseE25+bd3e0vzHV+ICkl3Wsr+Vz362+Fc8/oqWDk0PZh6fjxpmTADBihj1UAgBYJYIbAEYMwQ0AI4bgBoARQ3ADwIghuAFgxBDcADBiCG4AGDH/D3M5v2xeLs4UAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(costs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1e4af297b38>]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGDRJREFUeJzt3X1wHPd93/H3d+/weACIR0IkAD6KlKyQenBRm7Lc1lZsV07keDrjtnaeXDeN6ladyB13MknajsfTmU7bZOKkY8eJYruup64dNXYchfbYTmJZdmZk2ZBkUZRkSyIpPlMACRIEiMe7+/aP2wOOIEgcKRz2dvfzmsHs3t7e3lc7y8/99Nvf7pq7IyIi8RFEXYCIiFwfBbeISMwouEVEYkbBLSISMwpuEZGYUXCLiMSMgltEJGYU3CIiMaPgFhGJmWwtNtrb2+vbtm2rxaZFRBLpqaeeOuvufdWsW5Pg3rZtGyMjI7XYtIhIIpnZ0WrXVVeJiEjMKLhFRGJGwS0iEjMKbhGRmFFwi4jEjIJbRCRmFNwiIjFTN8FdKDqf/M7LPP7SWNSliIjUtboJ7kxg/Mn3DvM3L7wWdSkiInWtboIbYKCzhZMXZqIuQ0SkrtVVcA92tXDyvIJbRORa6iq4BzpbOKUWt4jINdVXcHe1MDmXZ2JmIepSRETqVn0Fd2crgLpLRESuob6Cu6sFQCcoRUSuob6CuzMM7vPTEVciIlK/6iq4e9saacoGanGLiFxDXQW3mWkst4jIKuoquKHUz62TkyIiV1d/wa0Wt4jINdVlcJ+dmmd2oRB1KSIidan+gltDAkVErilbzUpm9iowCRSAvLsP16qgpSGBM+zsa6vV14iIxFZVwR16u7ufrVklIbW4RUSure66Sm7qaCYwXfYuInI11Qa3A982s6fM7IGVVjCzB8xsxMxGxsZu/Ck22UzATR3NanGLiFxFtcF9j7u/EXg38KCZ/cPlK7j7w+4+7O7DfX19r6uogS4NCRQRuZqqgtvdT4XTUeAvgDfVsqiBTl2EIyJyNasGt5nlzKy9PA+8CzhYy6IGulo4c3GWfKFYy68REYmlakaV9AN/YWbl9f+vu3+zlkUNdLZSKDqvTc4tDg8UEZGSVYPb3Q8Dd6xDLYsWhwSen1Fwi4gsU3fDAaHiIpwLui+3iMhy9R3cOkEpInKFugzulsYMPblGDQkUEVlBXQY3lPq5T6jFLSJyhfoNbt2XW0RkRfUd3OdncPeoSxERqSt1G9xD3a3M5YucnZqPuhQRkbpSt8E9GI7lPnFeQwJFRCrVcXC3AnBcJyhFRC5Tx8GtFreIyErqNrhzTVm6c40cH1eLW0SkUt0GN8BQV4ta3CIiy9R1cA92teoiHBGRZeo7uLtLY7mLRY3lFhEpq+/g7mplvlBkbGou6lJEROpGXQf3UDiy5Pi4+rlFRMrqOrjLY7nVzy0isqTOg1stbhGR5eo6uJsbMvS1N6nFLSJSoa6DG0qt7uMayy0isqjug3tIY7lFRC5T98E92NXCqQszFDSWW0QEiEFwD3W3ki86Zy7ORl2KiEhdqPvgXrxLoEaWiIgAMQjuId2XW0TkMnUf3Js6mzHTfblFRMrqPribshlu6mjWyBIRkVDdBzeEY7nVxy0iAlxHcJtZxsyeMbP9tSxoJbovt4jIkutpcT8EvFirQq5lqKuF0xMzLBSKUXy9iEhdqSq4zWwQ+HngM7UtZ2WDXa0UHc5MaCy3iEi1Le4/AH4TuGqT18weMLMRMxsZGxtbk+LKBrt1l0ARkbJVg9vM7gdG3f2pa63n7g+7+7C7D/f19a1ZgVA5llvBLSJSTYv7HuAXzOxV4MvAvWb2f2pa1TKbNjSTDYxjanGLiKwe3O7+2+4+6O7bgPcD33H3X655ZRWymYCBrhaOnlNwi4jEYhw3wJbuVvVxi4hwncHt7t919/trVcy1bOlu5aiCW0QkPi3urT2tXJheYGJmIepSREQiFZvg3tKdAzQkUEQkRsFdGhKoE5QiknbxCe6eMLjHL0VciYhItGIT3G1NWXpyjeoqEZHUi01wQ6nVra4SEUm7WAX31m4Ft4hIrIJ7S3crpydmmM/r9q4ikl7xCu6eHEWHkxf0UAURSa94BffikECNLBGR9IpVcG8NhwRqZImIpFmsgntjexNN2UAnKEUk1WIV3Gamm02JSOrFKrih1F2irhIRSbPYBfeW7hzHxqdx96hLERGJRAyDu4Xp+QJjU3NRlyIiEonYBffWHt3eVUTSLXbBPaTbu4pIysUwuFswU3CLSHrFLribshk2dTRzTF0lIpJSsQtuKHWX6LJ3EUmrWAb39t6cukpEJLViG9znLs0zMa0nvotI+sQ2uAGOqLtERFIolsG9oy8M7rNTEVciIrL+YhncQ92tBAZHxtTiFpH0iWVwN2UzDHa1cvisgltE0mfV4DazZjP7oZk9a2bPm9nH16Ow1WzvzXFEwS0iKVRNi3sOuNfd7wDuBO4zs321LWt15eDWXQJFJG1WDW4vKZ8FbAj/Ik/LHX05pucLjE7qLoEiki5V9XGbWcbMfgyMAn/t7k/WtqzVlYcEHtYJShFJmaqC290L7n4nMAi8ycz2LF/HzB4wsxEzGxkbG1vrOq+wOJZb/dwikjLXNarE3S8A3wXuW+G9h9192N2H+/r61qi8q9u8oYXGbKCx3CKSOtWMKukzs85wvgV4B/CTWhe2miAwtvdoZImIpE+2inU2Af/bzDKUgv4Rd99f27Kqs703x0ujk1GXISKyrlYNbnc/ANy1DrVct+19Of7mxdfIF4pkM7G8lkhE5LrFOu229+bIF50T52eiLkVEZN3EOrh3aGSJiKRQrIN7cSy3gltEUiTWwd2da6SjOashgSKSKrEObjNje1+bukpEJFViHdxQ6ufWfblFJE1iH9zbe3OcmphlZr4QdSkiIusiEcEN8KqePykiKRH74N7Z1wbAy6M6QSki6RD74N7RlyMwePk1XfouIukQ++BubsiwrSfHSwpuEUmJ2Ac3wO7+dl5+TV0lIpIOCQnuNl49d4nZBY0sEZHkS0Rw7+pvp+h6jJmIpEMignt3fzuA+rlFJBUSEdzbe3NkA1Nwi0gqJCK4G7NB6Wk4OkEpIimQiOCGcGSJHmMmIimQmODe1d/GsfFp3bNERBIvMcG9u78ddzg0pu4SEUm2BAV36Z4lOkEpIkmXmODe2pOjMRPwUwW3iCRcYoK7IROwoy+nS99FJPESE9xQuoJSXSUiknSJCu7dG9s4cX6GS3P5qEsREamZRAX3rvDS91f0UAURSbBEBfctN+meJSKSfIkK7i3drTRlAz3GTEQSbdXgNrMhM3vMzF40s+fN7KH1KOxGZAJjV38bL5y6GHUpIiI1U02LOw981N3fAOwDHjSz22pb1o3bO9DJcycncPeoSxERqYlVg9vdT7v70+H8JPAiMFDrwm7U3oENTMwscHx8JupSRERq4rr6uM1sG3AX8GQtilkLtw9uAOC5kxMRVyIiUhtVB7eZtQFfAT7i7ld0IpvZA2Y2YmYjY2Nja1njddnd305jJuDAyQuR1SAiUktVBbeZNVAK7S+6+1dXWsfdH3b3YXcf7uvrW8sar0tjNuDWTe0cVItbRBKqmlElBnwWeNHdf7/2Jb1+ewY28NwJnaAUkWSqpsV9D/ArwL1m9uPw7+dqXNfrcvvABi7O5jk2Ph11KSIiay672gru/neArUMta2ZveILywIkJtvbkIq5GRGRtJerKybLd/e00ZgONLBGRREpkcDdkAt6wqYPnTii4RSR5EhncAHsHOjh4coJiUScoRSRZEhvctw90MjmX56hOUIpIwiQ2uPcMlE9Q6kIcEUmWxAb3rv42mrKBLsQRkcRJbHCXT1Ae0AlKEUmYxAY3lG44pROUIpI0iQ7uO4c6uTRf4Cdn9CgzEUmORAf3vh09ADxx+FzElYiIrJ1EB/fmzha29rTyxCEFt4gkR6KDG+DuHT08eeQcBfVzi0hCJD+4d/YwOZvn+VMaXSIiyZD84C73c6u7REQSIvHBvbGjmZ19OZ2gFJHESHxwQ6m75EdHxlkoFKMuRUTkdUtHcO/o5dJ8QffnFpFESEVw79vRDaifW0SSIRXB3dPWxC397QpuEUmEVAQ3lPq5R46OM5cvRF2KiMjrkprg3rejh9mFIs8eVz+3iMRbioK7GzP4/stjUZciIvK6pCa4O1sb2be9h68fOI27Ln8XkfhKTXADvOeOzRw+e4nnT12MuhQRkRuWquC+b89NZAPjrw6ciroUEZEblqrg7s418tZdvex/Vt0lIhJfqQpugPfcvpmTF2Z4+pie/i4i8ZS64H7nz/TTmA3Yr+4SEYmpVYPbzD5nZqNmdnA9Cqq1juYG3n5LH18/cFoPVxCRWKqmxf154L4a17Gu3nPHZkYn5/jhkfGoSxERuW6rBre7fw9IVMLde+tGWhszGl0iIrGUuj5ugNbGLO94Qz/7nz3F1Fw+6nJERK7LmgW3mT1gZiNmNjI2Vv+XlX/onm1cnM3zpSePRV2KiMh1WbPgdveH3X3Y3Yf7+vrWarM1c9eWLu65uYc//f5hZhd0x0ARiY9UdpWUPfi2mxmdnOMrT5+IuhQRkapVMxzwS8ATwC1mdsLMfq32Za2Pu3f2cOdQJ3/8+CHyeh6liMRENaNKPuDum9y9wd0H3f2z61HYejAzHnz7zRwfn2H/gdNRlyMiUpVUd5UA/OytG7mlv50/+u4rFHVBjojEQOqDOwiMf/v2nbz02pTGdYtILKQ+uAHuv30zdwx18vG/eoFzU3NRlyMick0KbiATGL/7vtuZnF3gY48+H3U5IiLXpOAO7e5v5zfu3cX+A6f55sEzUZcjInJVCu4KH37bTm7b1MF//suDXJiej7ocEZEVKbgrNGQC/sf7bmf80jz/8WsH9ZQcEalLCu5l9gxs4KPv2s3XD5zm977906jLERG5QjbqAurRv/lHOzk+Ps2nHjvEQGcrv/jmLVGXJCKySMG9AjPjv7x3D2cmZvlPX3uOmzY0ce+t/VGXJSICqKvkqrKZgE/+4hu5bXMHD37xGR5/qf5vVSsi6aDgvoZcU5bP/Yu/z7beHP/y8z/iyz/UvbtFJHoK7lVsbG/mkX+9j3tu7uW3vvocv/utn+ieJiISKQV3FdqbG/jsB4f5wJuG+NRjh/j1L4wwenE26rJEJKUU3FVqyAT813+yl4+95zb+7pWzvPMT3+Nrz5zUWG8RWXcK7utgZnzonu1846F/wM6+HB/5sx/z618Y4ZXRqahLE5EUUXDfgJ19bfy/D7+F3/m5W3ni0Dne9YnH+egjz3J8fDrq0kQkBawW/6s/PDzsIyMja77denRuao5Pf/cQX/jBUdydn9+7iV/at5XhrV2YWdTliUhMmNlT7j5c1boK7rVxZmKWP378EF95+gSTs3l297fxz4aHuG/PTQx2tUZdnojUOQV3hKbn8+x/9jRffPIoz56YAOBnNnfwrttu4q27erl9cAMNGfVQicjlFNx14tWzl/j2C2f45sEzPH3sAgCtjRn+3tYuhrd2s3ewgz0DG9jY3hxxpSISNQV3HTo3NccPj4zzg8PneOLwOV4enaK86ze2N7G7v52bN7Zx88Y2dvTmGOpuZXNnC5lA/eQiaXA9wa2bTK2TnrYm3r13E+/euwmAqbk8z5+c4LmTE7xw+iKHRqd4ZOQ40/OFxc80ZIyBzhY2bWhhU2czmze00L+hmY3tTWxsb6KvvYnetiaaGzJR/WeJSAQU3BFpa8ry5h09vHlHz+KyYtE5fXGWo2cvcXR8mmPj0xwfn+b0xCw/OHSOMxdnWelq+1xjhp62JrpaG+jKNdLV2siGlgY2tDTQ2Vqatjc30N6cpb05S0dzA21NWdqas+pvF4khBXcdCYJSC3ugs4W3rPB+vlBk/NI8o5NzjE7OMjY5x9mpec5NzXPu0hznpxc4NzXPK6NTTEwvMDmXX/U7G7MBbU1Zck0Zco1ZWhsztC5OM7Q0ZmhpyNLSGNDSkKH5sr+A5uzSfFN2adrUENCUDWjMBjRmArL6gRBZMwruGMlmAjZ2NLOxoxnYsOr6+UKRydk8EzMLTM7muTi7wMWZBabm8qW/2TxT83kuzeWZniswNZdnZqHApbk8Z6fmmJ4vMLNQYGa+wPR8fsXWfrUygdGYCYM8DPPGbEBDxsJp5bLS8tL0yvlsJqAhKE2zGaMhKE0rlzdkjGx5eWBkgtI2MkHpdTYTLC5fmgZkKtcPAoIAspVTQ+PzJXIK7gTLZoJS10mu8XVvy91ZKDiz+QKz8wVmF4rMLBSYDf/m8kXm8kVmFwrMh/Nz+dJ8+fV8oWI+fL1QsXyhUOTSXJ75QpF8wUvvF4os5J18sbxOaX6hEN09YjJhsGesFPJB+XW4LFPxOrBy8BuZADIWrl8xzZS3YaVtm125fPEz5c8FlOZt6XvK7wVW+h6rfC8IP2tLnwsqvq+0HS6bD6z8Hld+tmLd8nIrT2GxjuXrVK5Xft+Wbd8WP1fxmsu3gXHFNuHy7Sb5R1bBLVUxMxqzpdZxR3ND1OXg7uSLTj4M8nzBWQin5flC0VkolKdOoejkC0UWik6xWPp8IfwRKHrFtoq++Jli0Sl4+bPl+dI6xaJTKEKhWAyXc/l7Tvg9RQpFKIbbKU8L4Xtz+aV1l69TdBbXdffF71mad9wv33YxfK37n5Vc9oOy+MNw+TJW+PFZ+lFZ+uEo/xAEweXLwk3Qk2vikQ/fXfP/pqqC28zuA/4QyACfcff/VtOqRFZhZmEXCpQOS1muWHQcKgJ9KdSLxYp5d4rFq8x7+YdhaTuLPxThvPvSD4yz9H7ltPJHxb1UV/nHBi5fz1na5tI2lj5brpvFbbD4vYT/ve6lZcXF7bH0/eUal71XrmvxdbmOiu1X1r9YUzjvDu3N69MWXvVbzCwDfAp4J3AC+JGZPeruL9S6OBG5cUF4DYCuBUieak71vwl4xd0Pu/s88GXgvbUtS0RErqaa4B4Ajle8PhEuExGRCFQT3Cv9f9YVpz3M7AEzGzGzkbExPRFdRKRWqgnuE8BQxetB4NTyldz9YXcfdvfhvr6+tapPRESWqSa4fwTsMrPtZtYIvB94tLZliYjI1aw6qsTd82b274BvURp39Tl3f77mlYmIyIqqGnTo7t8AvlHjWkREpAq684+ISMzU5EEKZjYGHF3zDa+vXuBs1EXUEe2PK2mfXE7740rXs0+2untVIztqEtxJYGYj1T6NIg20P66kfXI57Y8r1WqfqKtERCRmFNwiIjGj4L66h6MuoM5of1xJ++Ry2h9Xqsk+UR+3iEjMqMUtIhIzqQ9uMxsys8fM7EUze97MHgqXd5vZX5vZy+G0K+pa15uZZczsGTPbH77ebmZPhvvkz8JbIKSCmXWa2Z+b2U/CY+XutB8jZvbvw38zB83sS2bWnKZjxMw+Z2ajZnawYtmKx4SV/E8ze8XMDpjZG1/Pd6c+uIE88FF3fwOwD3jQzG4Dfgv4W3ffBfxt+DptHgJerHj934FPhPvkPPBrkVQVjT8EvunutwJ3UNovqT1GzGwA+A1g2N33ULodxvtJ1zHyeeC+Zcuudky8G9gV/j0AfPp1fbOXH9mjP8L+/r+k9LSfnwKbwmWbgJ9GXds674fB8MC7F9hP6fa+Z4Fs+P7dwLeirnOd9kUHcITwnFDF8tQeIyzdp7+b0q0z9gP/OG3HCLANOLjaMQH8CfCBlda7kT+1uCuY2TbgLuBJoN/dTwOE043RVRaJPwB+EyiGr3uAC+6eD1+n6YEaO4Ax4H+FXUefMbMcKT5G3P0k8HvAMeA0MAE8RXqPkbKrHRNr+kAaBXfIzNqArwAfcfeLUdcTJTO7Hxh196cqF6+walqGJGWBNwKfdve7gEukqFtkJWHf7XuB7cBmIEepO2C5tBwjq1nTfz8KbsDMGiiF9hfd/avh4tfMbFP4/iZgNKr6InAP8Atm9iqlZ4zeS6kF3mlm5TtKrvhAjYQ6AZxw9yfD139OKcjTfIy8Azji7mPuvgB8FXgL6T1Gyq52TFT1QJpqpT64zcyAzwIvuvvvV7z1KPDBcP6DlPq+U8Hdf9vdB919G6UTTt9x918CHgPeF66Wmn3i7meA42Z2S7joZ4EXSPExQqmLZJ+ZtYb/hsr7JJXHSIWrHROPAr8aji7ZB0yUu1RuROovwDGztwLfB55jqT/3dyj1cz8CbKF0kP5Tdx+PpMgImdnbgP/g7veb2Q5KLfBu4Bngl919Lsr61ouZ3Ql8BmgEDgMfotTwSe0xYmYfB/45pZFZzwD/ilK/bSqOETP7EvA2SncAfA34GPA1Vjgmwh+3T1IahTINfMjdR274u9Me3CIicZP6rhIRkbhRcIuIxIyCW0QkZhTcIiIxo+AWEYkZBbeISMwouEVEYkbBLSISM/8fLzzBVt+t7asAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(costs[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1e4aecf9898>]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAFhFJREFUeJzt3X+QXeV93/H3V7srCbCQBJL4IQkkHLlGiZ2IUQyO7ZQa7ADJiLTj6aCQ1E4dM9OENqndNlAytHX+sp1JPWmobeK4djM2mDg/rPGQURyHpNMkUIQxmB9WvBYYrbHNArKMtEh7f3z7xz13dffuXe0Vurv3nsv7NbOz95zz7N7vPqvz0bPPee65kZlIkobLsn4XIEnqPcNdkoaQ4S5JQ8hwl6QhZLhL0hAy3CVpCBnukjSEDHdJGkKGuyQNodF+PfG6detyy5Yt/Xp6SSqlhx566PnMXL9Qu76F+5YtW9i3b1+/nl6SSikivt1NO6dlJGkIGe6SNIQMd0kaQguGe0R8KiKei4jH5jkeEfF7ETEeEY9GxGW9L1OSdCq6Gbl/GrjmJMevBbYVHzcBHzv9siRJp2PBcM/M/wO8eJIm1wP/OxvuB9ZExAW9KlCSdOp6Mee+ETjYsj1R7JMk9Ukv1rlHh30d37svIm6iMXXDRRdd1IOnlqSGZ16Y4k++OkEZ3jr0qkvP48c3r1nU5+hFuE8Am1u2NwHPdmqYmXcCdwLs3Llz8H8DAyAz+fIT3+fwy5Ulf+7xySPc9cAzlOBckXjpeBWA6DTcHDAbzl5ZinDfA9wcEXcDlwOHM/O7Pfi+Q+9YpcajE4ept6TnZ/7+aZ5+YWpme/Kl4zx/5Hg/yptx4+UXsWJ0pK81SN14y4+cy1WXntfvMgbCguEeEXcBVwLrImIC+C/AGEBmfhy4F7gOGAemgF9erGIH3fNHjnO8Wu+q7SMHf8B//rOv84OpuSPys1eOcvkl5wKwae0ZrBhdxm9cva0vAbv6zDHOXjm25M8r6fQsGO6ZuXuB4wn8Ws8qKqEfHqvwgXse4ctPfP+Uv/a6N5zPL15x8cz2SAQ/cdEaR8qSTkvfbhxWZpnJF7/27Mw8+Ef/6h85NFVh1YpRfvPa17N8pLtFSG/cvJrXn3/2YpYq6VXKcD8Fh6cqfGjvN/jiw9/h6HRt1rG3v34Dn3rPT/apMkmazXDvwvFqjWdemOIje/fzl098n5FlwS9cfhG/ftU2xkaWEcCaM52XljQ4XvXhXqnVqdXnX+v37RemuPGT9/P8kWkAfvp16/n9X9jhRUZJA+1VF+7/8K0XePjgIZZFcGhqmk/87YGuvu7qSzfwLy7bxFu3rTPYJQ28V0W4f+N7P+TIsSoXrjmD3X9w/5zj73vbVs45a8W8X/+Gjat567Z1i1miJPXUUIf7C0eO86uf/SoPPNW479mFq1cC8LEbL+N4tc7Dzxzi197+I2xYtbKfZUpSzw1tuE9NV3nbh+9jarrGmjPHWDG6jNdueA1v3baOd/7o+YwsC35+h/c3kzSchjLcn37+KFf+zt8A8LNvvIDf372DKMMNJySpR4bybfY++X8bF0l/6YqL+R83GOySXn2GauR+5HiVX/nMg9x/4EV+cstafvvnf6zfJUlSXwzVyP3Tf/cU9x9oXDz90QtX97kaSeqfoRm5H6vU+OhffROAz73vcnZsXtvniiSpf4Ym3MefO0K1nvzWz17KT73WNemSXt2GZlpm8qXGG1rsuMgRuyQNTbh/a/IIABtWzf9KU0l6tRiacP/8gwcZGwk2rjmj36VIUt8NTbh/7/Ax3rZtPcuWuaZdkoYi3I9Varx0vMrOLc63SxIMSbgfmmrca33tmcv7XIkkDYahCPfme5muPsP7rEsSDEm4v1y8n+kZYyN9rkSSBsNQhPuxSh2AFWND8eNI0mkbijQ8Vm2M3Fc6cpckYEjC/XilCPdRw12SYEjCvTkts9JpGUkChiTcjzstI0mzDEW4nxi5G+6SBEMT7s2R+1D8OJJ02oYiDWdG7l5QlSRgWMK9WmP5yDJvGiZJheEI90rNFzBJUouhSMRjlTornJKRpBlDEe7HKzUvpkpSi6FIxGPVmssgJanFcIR7pe7IXZJadJWIEXFNROyPiPGIuKXD8Ysi4r6IeDgiHo2I63pf6vyOV2sug5SkFguGe0SMAHcA1wLbgd0Rsb2t2W8B92TmDuAG4H/2utCTaYzcDXdJaupm5P4mYDwzD2TmNHA3cH1bmwTOLh6vBp7tXYkLO16tsWLUaRlJahrtos1G4GDL9gRweVub/wr8ZUT8W+As4OqeVNelSjVZbrhL0oxuErHTyz6zbXs38OnM3ARcB/xRRMz53hFxU0Tsi4h9k5OTp17tPKZrdcZGDHdJauomESeAzS3bm5g77fJe4B6AzPwHYCWwrv0bZeadmbkzM3euX7/+lVXcwXTVcJekVt0k4oPAtojYGhHLaVww3dPW5hngKoCIuJRGuPduaL6A6VrdaRlJarFgImZmFbgZ2As8SWNVzOMR8cGI2FU0+wDwvoh4BLgLeE9mtk/dLJpKrc7yEW8aJklN3VxQJTPvBe5t23d7y+MngLf0trTuVZyWkaRZhiIRnZaRpNlKn4iZSaWWjtwlqUXpE3G61ngXJkfuknRC6ROxUmtct13uyF2SZpQ+ESvVxsh9zNUykjSj9OF+YlrGG4dJUlP5w92RuyTNUf5w94KqJM1R+kSsNMPdC6qSNKP0iXhiWqb0P4ok9UzpE7HitIwkzVH6RJyuNta5O3KXpBNKn4gnLqi6WkaSmkof7s0XMS0fcZ27JDWVPtybI/cxR+6SNKP04e5SSEmaq/SJeNylkJI0R+kTsTlyN9wl6YTSJ2Kt3lgKOeq9ZSRpRunDvXk/97Flpf9RJKlnSp+ItXpjWmbEkbskzSh9uDdH7qPLDHdJaip9uM/MuRvukjSj9OFeLVbLjBjukjSj/OFeT0aXBRGGuyQ1DUe4ezFVkmYpf7jXklGXQUrSLKVPxWq97shdktoMQbinK2UkqU3pw73mtIwkzVH6VKzU6y6DlKQ2pQ/3Wj0Zc85dkmYpfbhXa+nIXZLalD/c63Xv5S5JbUqfio7cJWmu8od7PRl15C5Js3SVihFxTUTsj4jxiLhlnjb/MiKeiIjHI+JzvS1zftV63XXuktRmdKEGETEC3AG8A5gAHoyIPZn5REubbcCtwFsy81BEbFisgts1bj9guEtSq25G7m8CxjPzQGZOA3cD17e1eR9wR2YeAsjM53pb5vy8cZgkzdVNuG8EDrZsTxT7Wr0OeF1E/F1E3B8R13T6RhFxU0Tsi4h9k5OTr6ziNo3bDzjnLkmtuknFTsPibNseBbYBVwK7gU9GxJo5X5R5Z2buzMyd69evP9VaO6rWnHOXpHbdhPsEsLllexPwbIc2X8zMSmY+BeynEfaLrlZPlhnukjRLN+H+ILAtIrZGxHLgBmBPW5s/B/4ZQESsozFNc6CXhc6nnsmI78IkSbMsGO6ZWQVuBvYCTwL3ZObjEfHBiNhVNNsLvBARTwD3Af8xM19YrKJb1eq+iEmS2i24FBIgM+8F7m3bd3vL4wTeX3wsqUxw4C5Js5V+mUktHblLUrvyh3vdOXdJalf6cK+7WkaS5ih9uNdcLSNJc5Q/3Os4cpekNqUP98zEbJek2Uof7q6WkaS5yh/u9WSZc+6SNEvpw73uK1QlaY7Sh7vTMpI0V+nDvZ44LSNJbcof7nVXy0hSu9KHu9MykjRXqcM9M0mnZSRpjlKHe63eeLc/R+6SNFu5wz0Nd0nqpNThXmS70zKS1KbU4d6clnHgLkmzlTvcnZaRpI5KHe71mZG74S5JrUod7q6WkaTOyh3uxbSMb9YhSbOVOtybq2V8mz1Jmq3U4e5qGUnqbDjC3XSXpFlKHe715lJIp2UkaZZSh7urZSSps1KHe93VMpLUUcnDvfHZaRlJmq3U4e5qGUnqbDjC3XSXpFlKHe6ulpGkzkod7tXmapkRw12SWpU73GuNcB9bVuofQ5J6rtSpWK3VARh15C5Js5Q63CvFtMyY4S5Js3QV7hFxTUTsj4jxiLjlJO3eFREZETt7V+L8ZkbuTstI0iwLpmJEjAB3ANcC24HdEbG9Q7tVwL8DHuh1kfOpFHPuTstI0mzdDHnfBIxn5oHMnAbuBq7v0O63gQ8Dx3pY30lV642R+9iII3dJatVNKm4EDrZsTxT7ZkTEDmBzZn7pZN8oIm6KiH0RsW9ycvKUi23XXC0z6ouYJGmWbsK9U3LmzMGIZcB/Bz6w0DfKzDszc2dm7ly/fn33Vc6jUnPkLkmddJOKE8Dmlu1NwLMt26uAHwP+JiKeBq4A9izFRdXmi5icc5ek2boJ9weBbRGxNSKWAzcAe5oHM/NwZq7LzC2ZuQW4H9iVmfsWpeIWrpaRpM4WTMXMrAI3A3uBJ4F7MvPxiPhgROxa7AJPprlaxnXukjTbaDeNMvNe4N62fbfP0/bK0y+rO83VMqPOuUvSLKVOxYqrZSSpo1KH+8yNwxy5S9IspU7Far1OhG+QLUntSh3ulVp6u19J6qDUyVit1V3jLkkdlDvc6+nFVEnqoNThXqnVvZgqSR2UOhmrtXRaRpI6KHW4V+p1bz0gSR2UOhmrtfTWA5LUQbnDvV731gOS1EGpk7FSc7WMJHVS6nCvulpGkjoqdTJW666WkaROSh3ulVrd2w9IUgelTkbXuUtSZ6UO90o9XS0jSR2UOhmrtTpjrpaRpDlKHu5Oy0hSJ6UO94ovYpKkjkqdjNVaOi0jSR2UPNwduUtSJ6VOxkrdG4dJUielDvdqzVv+SlInpU5GV8tIUmelDvdK3RuHSVInpU7Gqrf8laSOShvumVncFbK0P4IkLZrSJmO1ngCuc5ekDsob7rVGuDtyl6S5SpuMlXodwHXuktRBacN9ZuTutIwkzVHicG+M3J2WkaS5SpuMleYFVadlJGmO0ob7zMjd2w9I0hylTcbKzGoZR+6S1K6rcI+IayJif0SMR8QtHY6/PyKeiIhHI+IrEXFx70udrTqzWqa0/z9J0qJZMBkjYgS4A7gW2A7sjojtbc0eBnZm5huBLwAf7nWh7VwtI0nz62bY+yZgPDMPZOY0cDdwfWuDzLwvM6eKzfuBTb0tc65KzZG7JM2nm2TcCBxs2Z4o9s3nvcBfdDoQETdFxL6I2Dc5Odl9lR28XKkBsGLMcJekdt0kY6d5j+zYMOIXgZ3ARzodz8w7M3NnZu5cv35991V2cOhoBYBzz1pxWt9HkobRaBdtJoDNLdubgGfbG0XE1cBtwD/NzOO9KW9+L05NA7D2rLHFfipJKp1uRu4PAtsiYmtELAduAPa0NoiIHcAngF2Z+Vzvy5zr0NEi3M9cvhRPJ0mlsmC4Z2YVuBnYCzwJ3JOZj0fEByNiV9HsI8BrgD+OiK9FxJ55vl3PvHh0mlUrR72gKkkddDMtQ2beC9zbtu/2lsdX97iuBR2amuacsxy1S1InpR32vnh02ikZSZpHacPdkbskza+84X604shdkuZR2nB/8eg057gMUpI6KmW4vzxd4+VKjbVOy0hSR6UM90PFC5jOcVpGkjoqZbg/OnEYwJG7JM2jlOE+/txLALz+/FV9rkSSBlMpw/35I41Xp1587ln9LkWSBlIpw338uSNcuPqMfpchSQOrlOH++LOHuezitf0uQ5IGVunCfWq6yqGpCpvWOnKXpPmULtyf/cHLAGxcY7hL0nxKF+5f/05jGeTWdV5MlaT5lC7cV60Y4x3bz+MNG1f3uxRJGlhd3c99kFy9/Tyu3n5ev8uQpIFWupG7JGlhhrskDSHDXZKGkOEuSUPIcJekIWS4S9IQMtwlaQgZ7pI0hCIz+/PEEZPAt1/hl68Dnu9hOYvBGk/foNcHg1/joNcH1niqLs7M9Qs16lu4n46I2JeZO/tdx8lY4+kb9Ppg8Gsc9PrAGheL0zKSNIQMd0kaQmUN9zv7XUAXrPH0DXp9MPg1Dnp9YI2LopRz7pKkkyvryF2SdBKlC/eIuCYi9kfEeETc0qcaNkfEfRHxZEQ8HhG/Xuw/JyK+HBHfLD6vLfZHRPxeUfOjEXHZEtY6EhEPR8SXiu2tEfFAUePnI2J5sX9FsT1eHN+yBLWtiYgvRMQ3ir5886D1YUT8++J3/FhE3BURK/vdhxHxqYh4LiIea9l3yv0WEe8u2n8zIt69yPV9pPg9PxoRfxYRa1qO3VrUtz8ifqZl/6Kd651qbDn2HyIiI2Jdsb3kfdgTmVmaD2AE+BZwCbAceATY3oc6LgAuKx6vAv4R2A58GLil2H8L8KHi8XXAXwABXAE8sIS1vh/4HPClYvse4Ibi8ceBf1M8/lXg48XjG4DPL0FtnwF+pXi8HFgzSH0IbASeAs5o6bv39LsPgZ8GLgMea9l3Sv0GnAMcKD6vLR6vXcT63gmMFo8/1FLf9uI8XgFsLc7vkcU+1zvVWOzfDOyl8Rqcdf3qw578jP0u4BR/IW8G9rZs3wrcOgB1fRF4B7AfuKDYdwGwv3j8CWB3S/uZdotc1ybgK8DbgS8V/zifbznJZvqz+Af95uLxaNEuFrG2s4vgjLb9A9OHNML9YHHyjhZ9+DOD0IfAlrbwPKV+A3YDn2jZP6tdr+trO/bPgc8Wj2edw80+XIpzvVONwBeAHwee5kS496UPT/ejbNMyzZOtaaLY1zfFn947gAeA8zLzuwDF5w1Fs37V/VHgPwH1Yvtc4AeZWe1Qx0yNxfHDRfvFcgkwCfyvYtrokxFxFgPUh5n5HeB3gGeA79Lok4cYnD5sdar91s9z6V/TGAlzkjqWvL6I2AV8JzMfaTs0MDWeirKFe3TY17flPhHxGuBPgN/IzB+erGmHfYtad0T8HPBcZj7UZR1LXeMojT+LP5aZO4CjNKYT5tOPPlwLXE9juuBC4Czg2pPUMVD/Pgvz1dSXWiPiNqAKfLa5a546lrS+iDgTuA24vdPheWoZxN/3jLKF+wSNObGmTcCz/SgkIsZoBPtnM/NPi93fj4gLiuMXAM8V+/tR91uAXRHxNHA3jamZjwJrIqL5xuitdczUWBxfDby4iPVNABOZ+UCx/QUaYT9IfXg18FRmTmZmBfhT4KcYnD5sdar9tuT9WVxw/DngxizmMQaovtfS+E/8keKc2QR8NSLOH6AaT0nZwv1BYFuxWmE5jYtWe5a6iIgI4A+BJzPzd1sO7QGaV8zfTWMuvrn/XxVX3a8ADjf/hF4smXlrZm7KzC00+umvM/NG4D7gXfPU2Kz9XUX7RRuFZOb3gIMR8U+KXVcBTzBAfUhjOuaKiDiz+J03axyIPmxzqv22F3hnRKwt/kJ5Z7FvUUTENcBvArsyc6qt7huKlUZbgW3A/2OJz/XM/HpmbsjMLcU5M0Fj0cT3GJA+PGX9nvR/BRdBrqOxOuVbwG19quGtNP78ehT4WvFxHY351a8A3yw+n1O0D+COouavAzuXuN4rObFa5hIaJ8848MfAimL/ymJ7vDh+yRLU9RPAvqIf/5zGioOB6kPgvwHfAB4D/ojGqo6+9iFwF41rABUaIfTeV9JvNOa+x4uPX17k+sZpzE83z5ePt7S/rahvP3Bty/5FO9c71dh2/GlOXFBd8j7sxYevUJWkIVS2aRlJUhcMd0kaQoa7JA0hw12ShpDhLklDyHCXpCFkuEvSEDLcJWkI/X+qxFf7xhwykgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(accs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1e4aee887f0>]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGpFJREFUeJzt3Xt0nPV95/H3V5J1sSVZtiVfdLEtg8F3AzHGCSkBQhpMg2nSNEBCUhIOZLNJ093mZA/ZNDktPT2bpHtSmi5lQ8lt2SQskAAOBROuCTTYYPBNvmEhG1k3S5askS1Z1mW++4fGjiJkNJZm9Mw883mdo2M9z/zQfJ/zSB+e+f6ei7k7IiISLllBFyAiIomncBcRCSGFu4hICCncRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhlBPUG5eWlvrChQuDensRkbT0+uuvH3X3srHGBRbuCxcuZOvWrUG9vYhIWjKzt+MZp7aMiEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiEU2HnuIiLpamAwSt3RbnY3RTh0tIdzfVzpB5fOYXVVSZKqG6JwF5EJc3feqO/kSFdvbBkaO3vY09TF7qYu2rv7Jq0WA+bPmsry8mKWzZtOydQp7zreHZojJ8/U2nbi1JjvceLUAH0D0d+/p51bjbOL8xXuIulsYDBKU2cvg+N8EH3Unfr2HmoaI+xp7qKrt3/UcSUFuSwrL2Z5eTHzZ07FzjVtxql/MMoze47w8NbDHGrvecfr86bns7y8mMsWzcSYnJoGok5d2wke39bE/91cH/d/N6c4j+Xl07m0esaYtRbkZrNkbhHLy6dzXtk0crJTr8OtcBc5R8d7+9nbfJyaxgiNnSdHHdPTN8Ce5uPsa+7i1LAjvIlYOGsqpYV5o762qyPCv+9qTsj7jMdl1TP5y6sXs7yi+EwwlhbmMuss9U6GaNRp7DxJT9/gmGNnTsulrCi4WpNB4S4ZIxp13u7oYW9zF92nBt51rANHIr3UNEWGPqof//1H9eFhXTAlm+ysdx7l5eZkceGcIj69bgEXzCkiN2f8R3blJQUsnVdEUf67txciJ/vZ29xFS6R33O91rsxgVWUJ1aXTJu0945WVZVTNnBp0GYFRuEsoHOvu47HtjWyqaaF3lCPlwWiUQ0d7ODFGqI9UXTqN1VUlVJYUcPqTenH+FJbNG2qBzC7OT0T5CTG9YArrFs0KugxJEQp3mTTuzmB07N5zlhlZoxwNjzQYdV460MbDWxt4Zs8R+gajLJtXPOrHazN4z/wZLC+fztJ5xWNOsgHMmJZLYZ7+RCQ96TdXkq6+vYdHXj/ML95oPGuPeri8nCyWxo6MF5UVkj1Kzrd0neLx7Y00R3qZMXUKn1o3nxsvrWLJ3OIkbIFI+lG4y7h0nxrgQOsJevuHJquiUY+d99vF3uauM+2P0+vN4IrFZdx4adWY50wc6+lnd1OEjTuaON47ehsly+CKC8r45keW8cGlcybU0xYJI4W7xO1IVy//+PR+3qg/xsGj3Yx2dl9xfg7Ly6dTUVJwZt3HLqngY5dUUj5sXTzcncjJ/lHfJzcni2lqmYiclf46JC6/qz3Klx/cRvepQf5ocSk3rK5g6bwiCvN//ytUNWMqlTMKEnaOtZlRMjU3IT9LJNMo3OVd9fYPcv9LdXz3mTdZVFbIz2+/hMVzioIuS0TGoHAXYKgF0nr81JmzWdqOn+KXbzTw2PYmIif7uX51Od/62Eq1QkTShP5SM1zb8VM8uq2Bh7Y2UNt64g9ey83J4sPL53LjmiouP3/WpF3SLiITp3DPYPf99i2+s2k/A1HnkvklfOMjyyjMywYgf0o2H7igTD1vkTSlcM9A7s7dzx7gn587wIeXz+GrH76Q82erjy4SJgr3DOPu/I+n9nHfb+v48/dU8q0/WzXqvVFEJL0p3DPMvb95i/t+W8en1y3g7zYsj+syfxFJPwr3DNLUeZLvPXeAP142h7tuWK4JUpEQ0zXbGeTbm/YRdfjGR5Yp2EVCTuGeIbYe6uDx7U18/opFGX2Pa5FMoXDPANGo87e/2s3c4ny+cOV5QZcjIpNA4Z4BHnmjgZrGLr523RKm5mqaRSQTKNxDbmAwyr88f4DVldPZsLo86HJEZJIo3EPuiZ3NHO44yRevOl+TqCIZROEeYtGoc++Lb7F4diHXLJ0TdDkiMokU7iH2/L5W9h85zn++6jxdrCSSYRTuIeXu3PNiLZUzCrh+lXrtIplG4R5Sm+s62FbfyeevWEROtnazSKbRX31I/euLtZQW5vLna6qCLkVEAqBwD6FdDRFeOnCUz72/mvwp2UGXIyIBULiH0L2/qaUoP4db1i0IuhQRCYjCPWRqW0/wVE0Ln3nvAorzpwRdjogEROEeMt//zVvkZmfx2curgy5FRAIUV7ib2bVmtt/Mas3szlFen29mL5jZNjPbaWbXJb5UGUtj50ke3dbIzWvnU1qYF3Q5IhKgMcPdzLKBe4D1wDLgZjNbNmLY3wAPufvFwE3Avya6UBnb/S/VAXD7FYsCrkREghbPkftaoNbd69y9D3gQuGHEGAeKY99PB5oSV6LEo28gyqPbGlm/ch4VJQVBlyMiAYvn/q8VwOFhyw3AZSPG/C3wazP7S2AacE1CqpO4vVzbRmdPP396ka5GFZH4jtxHuymJj1i+Gfixu1cC1wEPmNk7fraZ3WFmW81sa1tb27lXK2e1cXsT0wum8EeLy4IuRURSQDzh3gAMv8yxkne2XW4DHgJw91eAfKB05A9y9/vcfY27rykrUwglysm+QX695wjXrZxLbo5OgBKR+ML9NWCxmVWbWS5DE6YbR4ypBz4IYGZLGQp3HZpPkuf2HaGnb5Dr9TAOEYkZM9zdfQD4EvA0sJehs2J2m9ldZrYhNuwrwO1mtgP4OXCru49s3UiSbNzexOyiPC6rnhV0KSKSIuJ6oKa7Pwk8OWLdN4d9vwe4PLGlSTwiJ/t5cX8bt6xbQLbu2S4iMWrQprmnd7fQNxhlg86SEZFhFO5pbuP2JhbMmsrqyulBlyIiKUThnsZqW0/wcu1R/uySSj38WkT+gMI9jf3kd4fIzc7ik5fND7oUEUkxCvc0FTnZzy/eaGDDReW6SZiIvIPCPU099NphevoG+ezlC4MuRURSkMI9DQ0MRvnx7w6xtnomy8s1kSoi76RwT0PP7j1CY+dJPqejdhE5C4V7GvrRfxyioqSADy2bG3QpIpKiFO5ppjlyki0HO7jp0ipdkSoiZ6VwTzObaloAuG7VvIArEZFUpnBPM0/VtHDBnELOKysMuhQRSWEK9zTSeryX1w51sH6FjtpF5N0p3NPIr3cfwR3Wr9REqoi8O4V7GtlU08Ki0mlcOKco6FJEJMUp3NPEse4+Xqlr59oVc3WTMBEZk8I9TTyz5wiDUVe/XUTionBPE0/VNFM5o4AVFcVBlyIiaUDhngYiPf28XHuU9WrJiEicFO5pYNPuZvoHnetX61F6IhIfhXsa2LijiYWzprKyQneAFJH4KNxTXOvxXl55q50Nq8vVkhGRuCncU9y/72wm6qglIyLnROGe4jbuaGLJ3CIW68IlETkHCvcUdrijh231nWy4SEftInJuFO4pbOOOJgCuX6VwF5Fzo3BPYb/a0cQl80uomjk16FJEJM0o3FPUka5e9rUc1+0GRGRcFO4pamdDBIBLFpQEXImIpCOFe4ra1Rghy2DZPF24JCLnTuGeonY1dHL+7EIKcrODLkVE0pDCPQW5O7sau1hZoZaMiIyPwj0FHek6xdETp1ip2/uKyDgp3FPQrsahydSVleq3i8j4KNxT0K6GTk2misiEKNxT0K7GCItnF2kyVUTGTeGeYk5Ppq7QvdtFZAIU7immpauXoydOsUr9dhGZgLjC3cyuNbP9ZlZrZneeZcwnzGyPme02s58ltszMsSt2ZaqO3EVkInLGGmBm2cA9wIeABuA1M9vo7nuGjVkMfA243N2PmdnsZBUcdjVnrkzVaZAiMn7xHLmvBWrdvc7d+4AHgRtGjLkduMfdjwG4e2tiy8wcuxojXDBHk6kiMjHxhHsFcHjYckNs3XAXABeY2X+Y2WYzuzZRBWaSocnUiFoyIjJhY7ZlgNGeyuyj/JzFwJVAJfCSma1w984/+EFmdwB3AMyfP/+ciw27ocnUPlYq3EVkguI5cm8AqoYtVwJNo4x53N373f0gsJ+hsP8D7n6fu69x9zVlZWXjrTm0Tk+m6spUEZmoeML9NWCxmVWbWS5wE7BxxJjHgKsAzKyUoTZNXSILzQS7GiNkZ5kmU0VkwsYMd3cfAL4EPA3sBR5y991mdpeZbYgNexpoN7M9wAvAV929PVlFh9XQlamF5E/RZKqITEw8PXfc/UngyRHrvjnsewf+OvYl4+Du1DRGuOpCnUUqIhOnK1RTRHMkNpmqfruIJIDCPUWcvs2vToMUkURQuKeIXQ2aTBWRxFG4pwhNpopIIincU8DpyVRdvCQiiaJwTwFNkV7auzWZKiKJo3BPAWeuTNWRu4gkiMI9BdTErkxdqslUEUkQhXsK2KnJVBFJMIV7wE5PpuqxeiKSSAr3gDVFeuno1m1+RSSxFO4Be6v1BAAXzCkKuBIRCROFe8DqO3oAWDBrWsCViEiYKNwDVt/RQ25OFrOL8oIuRURCROEesPr2HqpmFJCVNdrTDEVExkfhHrD6jh7mz5wadBkiEjIK9wC5O4cV7iKSBAr3AHX29HP81ADzNZkqIgmmcA/Q6TNldOQuIommcA/Q2wp3EUkShXuADsfCvWpmQcCViEjYKNwDVN/eQ2lhHlNzc4IuRURCRuEeoPqOHhbMUktGRBJP4R4gneMuIsmicA9I30CU5shJqhTuIpIECveANHaeJOo6U0ZEkkPhHhCd4y4iyaRwD4jCXUSSSeEekMMdPeTpVr8ikiQK94DUt/dQNXOqbvUrIkmhcA/I2zoNUkSSSOEeAN3qV0SSTeEegGM9/Zw4NaBz3EUkaRTuAXi7vRuABQp3EUkShXsAdjZEALhwblHAlYhIWCncA7DlYDsVJQVqy4hI0ijcJ5m7s7mug8sWzQy6FBEJMYX7JDvQeoKO7j7WVc8KuhQRCbG4wt3MrjWz/WZWa2Z3vsu4j5uZm9maxJUYLlvq2gFYt0jhLiLJM2a4m1k2cA+wHlgG3Gxmy0YZVwR8GdiS6CLDZHNdB/Om5+vReiKSVPEcua8Fat29zt37gAeBG0YZ9/fAd4DeBNYXKu7OloPtrFs0CzPddkBEkieecK8ADg9bboitO8PMLgaq3P2JBNYWOm+1neDoiT4uq9ZkqogkVzzhPtohpp950SwL+CfgK2P+ILM7zGyrmW1ta2uLv8qQ2FzXAajfLiLJF0+4NwBVw5YrgaZhy0XACuBFMzsErAM2jjap6u73ufsad19TVlY2/qrT1Oa6duYU5+mh2CKSdPGE+2vAYjOrNrNc4CZg4+kX3T3i7qXuvtDdFwKbgQ3uvjUpFaepoX57h/rtIjIpxgx3dx8AvgQ8DewFHnL33WZ2l5ltSHaBYVF3tJu246fUkhGRSZETzyB3fxJ4csS6b55l7JUTLyt8th4a6rdfulCTqSKSfLpCdZJsPxyhOD+HRaXTgi5FRDKAwn2S7GzoZFVliR6rJyKTQuE+CXr7B9nXcpxVldODLkVEMoTCfRLsbupiMOqsrioJuhQRyRAK90mws6ETgNWVCncRmRwK90mwsyHC7KI85k7PD7oUEckQCvdJsONwp1oyIjKpFO5JFjnZT93RblZrMlVEJpHCPclqGocehr1K/XYRmUQK9yTbfnhoMlWnQYrIZFK4J9nOhk4WzppKydTcoEsRkQyicE+ynQ0RtWREZNIp3JOotauX5kivzpQRkUmncE+iHQ1Dk6k6U0ZEJpvCPYl+taOJorwcVlQo3EVkcinck6Ql0suTu5r5xKVV5E/JDrocEckwCvckeWDzIaLu3Pq+hUGXIiIZSOGeBL39g/xsSz3XLJ1D1Uw9DFtEJp/CPQke397IsZ5+Pnt5ddCliEiGUrgnmLvzw5cPsWRuEesW6XmpIhIMhXuCvfJWO/uPHOdzl1djpkfqiUgwFO4J9sgbDRTn57DhovKgSxGRDKZwT6C+gSjP7jnCh5bN1emPIhIohXsC/e6to3T1DrB+xdygSxGRDKdwT6BNNS0U5uXw/sWlQZciIhlO4Z4gA4NRnt7dwtVLZqslIyKBU7gnyKsHOzjW0891K9WSEZHgKdwT5KmaFgqmZPOBC2YHXYqIiMI9EaJRZ9PuFq68sIyCXLVkRCR4CvcEeL3+GG3HT7F+5bygSxERARTuCbGppoXc7CyuXqKWjIikBoV7Arywv5X3njeLwrycoEsREQEU7hP2dns3dW3dXHVhWdCliIicoXCfoBf2tQJw5YVqyYhI6lC4T9AL+9tYVDqNhaXTgi5FROQMhfsEnOwb5JW6dh21i0jKUbhPwCt1R+kbiHLVEvXbRSS1KNwn4IV9bRRMyWZttZ64JCKpJa5wN7NrzWy/mdWa2Z2jvP7XZrbHzHaa2XNmtiDxpaYWd+eF/a1cfn4peTm6KlVEUsuY4W5m2cA9wHpgGXCzmS0bMWwbsMbdVwGPAN9JdKGpprb1BA3HTqolIyIpKZ4j97VArbvXuXsf8CBww/AB7v6Cu/fEFjcDlYktM/U8p1MgRSSFxRPuFcDhYcsNsXVncxvw1ESKSnUvHzjK3c++ySXzS6goKQi6HBGRd4jnenkbZZ2POtDsFmAN8IGzvH4HcAfA/Pnz4ywxtTy39whf+OkbLCqdxn2fWRN0OSIio4rnyL0BqBq2XAk0jRxkZtcAXwc2uPup0X6Qu9/n7mvcfU1ZWfr1qjfuaOLzD7zOkrlFPHjHOkoL84IuSURkVPEcub8GLDazaqARuAn45PABZnYx8H3gWndvTXiVAesfjPKtp/bxg5cPsnbhTO6/dQ3F+VOCLktE5KzGDHd3HzCzLwFPA9nAD919t5ndBWx1943APwKFwMNmBlDv7huSWPekaYn08sWfvcHrbx/j1vct5L9ft5TcHF0eICKpLa571Lr7k8CTI9Z9c9j31yS4rpRQ23qCT92/meO9A3zv5ovZsLo86JJEROKiG5Cfxd7mLm65fwtmxi++8D6WzisOuiQRkbgp3Eexs6GTz/zwVfJzsvnp7ZdxXllh0CWJiJwTNY9HeHRbAzd+fzNF+Tk8/J/eq2AXkbSkI/eY3v5B7npiDz/bUs/a6pn8r5svZnZxftBliYiMi8IdONbdx60/fo0dhzv5/AcW8dU/vpCcbH2oEZH0lfHh3nq8l0/f/yoH27v537e8h2tXzA26JBGRCcvocG+OnORT/7aF5kgvP7r1Ui4/vzTokkREEiKjwv3Vgx18/dFddHT3AdDdN8CUrCweuG0taxbqgRsiEh4ZEe7uzr+9VMe3N+2nakYB61cOtV6yzbjx0vksK9c57CISLqEN94HBKHVHu6lpjPDEzmae39fK+hVz+c7HV1Gk+8KISMiFKtzdnS0HO3ho62E21bTQ0zcIQMGUbP7mT5Zy2/urid37RkQk1EIT7i8daOMbj9VwqL2HorwcNqwuZ231TFZUTGdR6TSd2igiGSXtwz0adf7l+Vrufu5Nzisr5LufWM36FfMoyNVDq0Ukc6VtuLefOMXupi5+8PJBfvNmGx+9uIJ/+OgKpuam7SaJiCRM2iXhg6/Wc/ezB2jp6gUgNyeLf/joCj65dr766SIiMWkX7rOL81i3aCbLy6ezvKKY5eXTmV6gs19ERIZLu3C/eskcrl4yJ+gyRERSmk4hEREJIYW7iEgIKdxFREJI4S4iEkIKdxGREFK4i4iEkMJdRCSEFO4iIiFk7h7MG5u1AW+P8z8vBY4msJx0kYnbnYnbDJm53Zm4zXDu273A3cvGGhRYuE+EmW119zVB1zHZMnG7M3GbITO3OxO3GZK33WrLiIiEkMJdRCSE0jXc7wu6gIBk4nZn4jZDZm53Jm4zJGm707LnLiIi7y5dj9xFRORdpF24m9m1ZrbfzGrN7M6g60kGM6sysxfMbK+Z7Tazv4qtn2lmz5jZgdi/M4KuNdHMLNvMtpnZE7HlajPbEtvm/2dmuUHXmGhmVmJmj5jZvtg+f2+G7Ov/Gvv9rjGzn5tZftj2t5n90Mxazaxm2LpR960N+V4s23aa2SUTee+0CnczywbuAdYDy4CbzWxZsFUlxQDwFXdfCqwDvhjbzjuB59x9MfBcbDls/grYO2z528A/xbb5GHBbIFUl1z8Dm9x9CbCaoe0P9b42swrgy8Aad18BZAM3Eb79/WPg2hHrzrZv1wOLY193APdO5I3TKtyBtUCtu9e5ex/wIHBDwDUlnLs3u/sbse+PM/THXsHQtv4kNuwnwJ8GU2FymFkl8CfA/bFlA64GHokNCeM2FwNXAD8AcPc+d+8k5Ps6JgcoMLMcYCrQTMj2t7v/FugYsfps+/YG4P/4kM1AiZnNG+97p1u4VwCHhy03xNaFlpktBC4GtgBz3L0Zhv4HAMwOrrKkuBv4b0A0tjwL6HT3gdhyGPf3IqAN+FGsHXW/mU0j5Pva3RuB/wnUMxTqEeB1wr+/4ez7NqH5lm7hbqOsC+3pPmZWCPwC+C/u3hV0PclkZh8BWt399eGrRxkatv2dA1wC3OvuFwPdhKwFM5pYn/kGoBooB6Yx1JYYKWz7+90k9Pc93cK9AagatlwJNAVUS1KZ2RSGgv2n7v7L2Oojpz+mxf5tDaq+JLgc2GBmhxhqt13N0JF8SexjO4RzfzcADe6+Jbb8CENhH+Z9DXANcNDd29y9H/gl8D7Cv7/h7Ps2ofmWbuH+GrA4NqOey9AEzMaAa0q4WK/5B8Bed//usJc2An8R+/4vgMcnu7ZkcfevuXuluy9kaL8+7+6fAl4APh4bFqptBnD3FuCwmV0YW/VBYA8h3tcx9cA6M5sa+30/vd2h3t8xZ9u3G4HPxM6aWQdETrdvxsXd0+oLuA54E3gL+HrQ9SRpG9/P0MexncD22Nd1DPWgnwMOxP6dGXStSdr+K4EnYt8vAl4FaoGHgbyg60vC9l4EbI3t78eAGZmwr4G/A/YBNcADQF7Y9jfwc4bmFPoZOjK/7Wz7lqG2zD2xbNvF0JlE435vXaEqIhJC6daWERGROCjcRURCSOEuIhJCCncRkRBSuIuIhJDCXUQkhBTuIiIhpHAXEQmh/w9GWtJOWf62ugAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(accs[:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply model to test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.0489982] [0.99246231]\n"
     ]
    }
   ],
   "source": [
    "print(costs[-1], accs[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.97660819])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy(theta, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = dataset.data\n",
    "y = dataset.target\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_with_ones, y, test_size = 0.3, random_state=12345)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LogisticRegression()\n",
    "lr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.992462311557789"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9766081871345029"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.16188044])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.intercept_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.16188044, -0.35607225, -0.51732699, -0.38050215, -0.45233913,\n",
       "       -0.34296444,  0.3498612 , -0.88317445, -0.99814281, -0.06963982,\n",
       "        0.27916329, -1.06875966,  0.09017138, -0.76342139, -0.75020754,\n",
       "        0.32578214,  1.15876839, -0.07459907, -0.49088402,  0.06566515,\n",
       "        0.7962668 , -0.79499813, -0.87404419, -0.75630483, -0.82259933,\n",
       "       -0.73091925,  0.13404571, -0.88450734, -0.83173228, -0.5178612 ,\n",
       "       -0.4865363 ])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.coef_[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_compare = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_compare['features'] = np.hstack((['bias'], data.columns[:-1]))\n",
    "data_compare['sklearn'] = lr.coef_[0]\n",
    "data_compare['mine'] = (theta) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>features</th>\n",
       "      <th>sklearn</th>\n",
       "      <th>mine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bias</td>\n",
       "      <td>0.161880</td>\n",
       "      <td>0.660577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mean radius</td>\n",
       "      <td>-0.356072</td>\n",
       "      <td>-0.417717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>mean texture</td>\n",
       "      <td>-0.517327</td>\n",
       "      <td>-0.686624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>mean perimeter</td>\n",
       "      <td>-0.380502</td>\n",
       "      <td>-0.465461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>mean area</td>\n",
       "      <td>-0.452339</td>\n",
       "      <td>-0.417080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>mean smoothness</td>\n",
       "      <td>-0.342964</td>\n",
       "      <td>-0.506114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>mean compactness</td>\n",
       "      <td>0.349861</td>\n",
       "      <td>-0.089967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>mean concavity</td>\n",
       "      <td>-0.883174</td>\n",
       "      <td>-0.952097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>mean concave points</td>\n",
       "      <td>-0.998143</td>\n",
       "      <td>-1.069773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>mean symmetry</td>\n",
       "      <td>-0.069640</td>\n",
       "      <td>-0.202518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>mean fractal dimension</td>\n",
       "      <td>0.279163</td>\n",
       "      <td>0.695040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>radius error</td>\n",
       "      <td>-1.068760</td>\n",
       "      <td>-1.099068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>texture error</td>\n",
       "      <td>0.090171</td>\n",
       "      <td>0.142689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>perimeter error</td>\n",
       "      <td>-0.763421</td>\n",
       "      <td>-0.837507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>area error</td>\n",
       "      <td>-0.750208</td>\n",
       "      <td>-0.615647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>smoothness error</td>\n",
       "      <td>0.325782</td>\n",
       "      <td>0.512581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>compactness error</td>\n",
       "      <td>1.158768</td>\n",
       "      <td>0.865116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>concavity error</td>\n",
       "      <td>-0.074599</td>\n",
       "      <td>0.216623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>concave points error</td>\n",
       "      <td>-0.490884</td>\n",
       "      <td>-0.379302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>symmetry error</td>\n",
       "      <td>0.065665</td>\n",
       "      <td>0.223310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>fractal dimension error</td>\n",
       "      <td>0.796267</td>\n",
       "      <td>0.870673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>worst radius</td>\n",
       "      <td>-0.794998</td>\n",
       "      <td>-0.779020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>worst texture</td>\n",
       "      <td>-0.874044</td>\n",
       "      <td>-0.937510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>worst perimeter</td>\n",
       "      <td>-0.756305</td>\n",
       "      <td>-0.746362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>worst area</td>\n",
       "      <td>-0.822599</td>\n",
       "      <td>-0.675477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>worst smoothness</td>\n",
       "      <td>-0.730919</td>\n",
       "      <td>-0.874154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>worst compactness</td>\n",
       "      <td>0.134046</td>\n",
       "      <td>-0.013241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>worst concavity</td>\n",
       "      <td>-0.884507</td>\n",
       "      <td>-0.808783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>worst concave points</td>\n",
       "      <td>-0.831732</td>\n",
       "      <td>-0.980588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>worst symmetry</td>\n",
       "      <td>-0.517861</td>\n",
       "      <td>-0.563465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>worst fractal dimension</td>\n",
       "      <td>-0.486536</td>\n",
       "      <td>-0.113248</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   features   sklearn      mine\n",
       "0                      bias  0.161880  0.660577\n",
       "1               mean radius -0.356072 -0.417717\n",
       "2              mean texture -0.517327 -0.686624\n",
       "3            mean perimeter -0.380502 -0.465461\n",
       "4                 mean area -0.452339 -0.417080\n",
       "5           mean smoothness -0.342964 -0.506114\n",
       "6          mean compactness  0.349861 -0.089967\n",
       "7            mean concavity -0.883174 -0.952097\n",
       "8       mean concave points -0.998143 -1.069773\n",
       "9             mean symmetry -0.069640 -0.202518\n",
       "10   mean fractal dimension  0.279163  0.695040\n",
       "11             radius error -1.068760 -1.099068\n",
       "12            texture error  0.090171  0.142689\n",
       "13          perimeter error -0.763421 -0.837507\n",
       "14               area error -0.750208 -0.615647\n",
       "15         smoothness error  0.325782  0.512581\n",
       "16        compactness error  1.158768  0.865116\n",
       "17          concavity error -0.074599  0.216623\n",
       "18     concave points error -0.490884 -0.379302\n",
       "19           symmetry error  0.065665  0.223310\n",
       "20  fractal dimension error  0.796267  0.870673\n",
       "21             worst radius -0.794998 -0.779020\n",
       "22            worst texture -0.874044 -0.937510\n",
       "23          worst perimeter -0.756305 -0.746362\n",
       "24               worst area -0.822599 -0.675477\n",
       "25         worst smoothness -0.730919 -0.874154\n",
       "26        worst compactness  0.134046 -0.013241\n",
       "27          worst concavity -0.884507 -0.808783\n",
       "28     worst concave points -0.831732 -0.980588\n",
       "29           worst symmetry -0.517861 -0.563465\n",
       "30  worst fractal dimension -0.486536 -0.113248"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_compare"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interpretatoin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\hat{y}=\\frac{1}{1 + e^{- \\theta x}}$ \n",
    "\n",
    "If $\\theta=0$, then $e^{- \\theta x}=1$ and $\\hat{y}=0.5$\n",
    "\n",
    "If $\\theta$ is positive, then $e^{- \\theta x}<1$ and $\\hat{y}>0.5$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "∴ $-\\theta x = \\frac{1-\\hat{y}}{\\hat{y}}$ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "|$\\theta$ | $e^{- \\theta x}$ | $\\hat{y}$|interpretatoin|\n",
    "|--:|--:|--:|--:|\n",
    "| 0 | 1 | 0.5 |x has no impect over y|\n",
    "| positive | decreate when x increase | increate |y increase when x increase|\n",
    "| negative | increate when x increase | decreate |y decrease when x increase|"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For feature 1 (mean radius), because it is negative, so the bigger mean radius is, the more likely y is 0, or to say the cancer is likely malignant.\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
